{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "### 1.GET UNIVERSAL CSV:\n",
    "### ui_df (user_id,item_id,time) sorted by time,\n",
    "### u_df (reviewerID,user_id) sorted by reviewerID,\n",
    "### i_df (asin,item_id) sorted by asin"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "data": {
      "text/plain": "'/data/Chester/preprocessing'"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "import pandas as pd\n",
    "import gzip\n",
    "import os\n",
    "import subprocess\n",
    "\n",
    "os.chdir(\"/data/Chester/preprocessing\")\n",
    "os.getcwd()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        yield eval(l)\n",
    "\n",
    "def get_df(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "DATASET=\"Beauty\"\n",
    "RAW_PATH=os.path.join('./',DATASET)\n",
    "DATA_FILE=\"reviews_{}_5.json.gz\".format(DATASET)\n",
    "META_FILE=\"meta_{}.json.gz\".format(DATASET)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading interaction data into ./Beauty\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 42.7M  100 42.7M    0     0  3215k      0  0:00:13  0:00:13 --:--:-- 7991k\n",
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading item metadata into ./Beauty\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100 94.5M  100 94.5M    0     0  5132k      0  0:00:18  0:00:18 --:--:-- 9161k\n"
     ]
    }
   ],
   "source": [
    "# download data if not exists\n",
    "\n",
    "if not os.path.exists(RAW_PATH):\n",
    "    subprocess.call('mkdir ' + RAW_PATH, shell=True)\n",
    "if not os.path.exists(os.path.join(RAW_PATH, DATA_FILE)):\n",
    "    print('Downloading interaction data into ' + RAW_PATH)\n",
    "    subprocess.call(\n",
    "        'cd {} && curl -O http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_{}_5.json.gz'\n",
    "        .format(RAW_PATH, DATASET), shell=True)\n",
    "if not os.path.exists(os.path.join(RAW_PATH, META_FILE)):\n",
    "    print('Downloading item metadata into ' + RAW_PATH)\n",
    "    subprocess.call(\n",
    "        'cd {} && curl -O http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/meta_{}.json.gz'\n",
    "        .format(RAW_PATH, DATASET), shell=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "data_df = get_df(os.path.join(RAW_PATH, DATA_FILE))\n",
    "data_df.head()\n",
    "ui_df=data_df.rename(columns={'asin':\"item_id\",'reviewerID':\"user_id\",'unixReviewTime':'time'})\n",
    "ui_df=ui_df[['user_id','item_id','time']]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(198502, 9) (198502, 3)\n"
     ]
    }
   ],
   "source": [
    "print(data_df.shape,ui_df.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "       reviewerID        asin reviewerName helpful  \\\n0  A1YJEY40YUW4SE  7806397051       Andrea  [3, 4]   \n1   A60XNB876KYML  7806397051   Jessica H.  [1, 1]   \n2  A3G6XNM240RMWA  7806397051        Karen  [0, 1]   \n3  A1PQFP6SAJ6D80  7806397051        Norah  [2, 2]   \n4  A38FVHZTNQ271F  7806397051    Nova Amor  [0, 0]   \n\n                                          reviewText  overall  \\\n0  Very oily and creamy. Not at all what I expect...      1.0   \n1  This palette was a decent price and I was look...      3.0   \n2  The texture of this concealer pallet is fantas...      4.0   \n3  I really can't tell what exactly this thing is...      2.0   \n4  It was a little smaller than I expected, but t...      3.0   \n\n                  summary  unixReviewTime   reviewTime  \n0  Don't waste your money      1391040000  01 30, 2014  \n1             OK Palette!      1397779200  04 18, 2014  \n2           great quality      1378425600   09 6, 2013  \n3  Do not work on my face      1386460800   12 8, 2013  \n4              It's okay.      1382140800  10 19, 2013  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>reviewerID</th>\n      <th>asin</th>\n      <th>reviewerName</th>\n      <th>helpful</th>\n      <th>reviewText</th>\n      <th>overall</th>\n      <th>summary</th>\n      <th>unixReviewTime</th>\n      <th>reviewTime</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>A1YJEY40YUW4SE</td>\n      <td>7806397051</td>\n      <td>Andrea</td>\n      <td>[3, 4]</td>\n      <td>Very oily and creamy. Not at all what I expect...</td>\n      <td>1.0</td>\n      <td>Don't waste your money</td>\n      <td>1391040000</td>\n      <td>01 30, 2014</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>A60XNB876KYML</td>\n      <td>7806397051</td>\n      <td>Jessica H.</td>\n      <td>[1, 1]</td>\n      <td>This palette was a decent price and I was look...</td>\n      <td>3.0</td>\n      <td>OK Palette!</td>\n      <td>1397779200</td>\n      <td>04 18, 2014</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>A3G6XNM240RMWA</td>\n      <td>7806397051</td>\n      <td>Karen</td>\n      <td>[0, 1]</td>\n      <td>The texture of this concealer pallet is fantas...</td>\n      <td>4.0</td>\n      <td>great quality</td>\n      <td>1378425600</td>\n      <td>09 6, 2013</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>A1PQFP6SAJ6D80</td>\n      <td>7806397051</td>\n      <td>Norah</td>\n      <td>[2, 2]</td>\n      <td>I really can't tell what exactly this thing is...</td>\n      <td>2.0</td>\n      <td>Do not work on my face</td>\n      <td>1386460800</td>\n      <td>12 8, 2013</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>A38FVHZTNQ271F</td>\n      <td>7806397051</td>\n      <td>Nova Amor</td>\n      <td>[0, 0]</td>\n      <td>It was a little smaller than I expected, but t...</td>\n      <td>3.0</td>\n      <td>It's okay.</td>\n      <td>1382140800</td>\n      <td>10 19, 2013</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ui_df=ui_df.sort_values(by=['time','user_id'],kind='mergesort').reset_index(drop=True)\n",
    "data_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "          user_id     item_id        time\n0  A281NPSIMI1C2R  B0000535UX  1023840000\n1  A281NPSIMI1C2R  B0000535UM  1024185600\n2  A281NPSIMI1C2R  B0000535UN  1024185600\n3   AWIF8AR75LL9L  B000065DK4  1036627200\n4  A281NPSIMI1C2R  B000052Y33  1052611200",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>item_id</th>\n      <th>time</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UX</td>\n      <td>1023840000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UM</td>\n      <td>1024185600</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UN</td>\n      <td>1024185600</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>AWIF8AR75LL9L</td>\n      <td>B000065DK4</td>\n      <td>1036627200</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B000052Y33</td>\n      <td>1052611200</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ui_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Check nan and duplicates data"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After dropped:(198502, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": "          user_id     item_id        time\n0  A281NPSIMI1C2R  B0000535UX  1023840000\n1  A281NPSIMI1C2R  B0000535UM  1024185600\n2  A281NPSIMI1C2R  B0000535UN  1024185600\n3   AWIF8AR75LL9L  B000065DK4  1036627200\n4  A281NPSIMI1C2R  B000052Y33  1052611200",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>item_id</th>\n      <th>time</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UX</td>\n      <td>1023840000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UM</td>\n      <td>1024185600</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UN</td>\n      <td>1024185600</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>AWIF8AR75LL9L</td>\n      <td>B000065DK4</td>\n      <td>1036627200</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B000052Y33</td>\n      <td>1052611200</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userID,itemID,timestamp=\"user_id\",\"item_id\",\"time\"\n",
    "ui_df.dropna(subset=[userID,itemID,timestamp],inplace=True)\n",
    "ui_df.drop_duplicates(subset=[userID,itemID,timestamp],inplace=True)\n",
    "print(f'After dropped:{ui_df.shape}')\n",
    "ui_df[:5]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Check K-core"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 illegal_ids_by_inter_num, field=user_id\n",
      "0 illegal_ids_by_inter_num, field=item_id\n",
      "k-core shape: (198502, 3)\n",
      "shape after k-core: (198502, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": "          user_id     item_id        time\n0  A281NPSIMI1C2R  B0000535UX  1023840000\n1  A281NPSIMI1C2R  B0000535UM  1024185600\n2  A281NPSIMI1C2R  B0000535UN  1024185600\n3   AWIF8AR75LL9L  B000065DK4  1036627200\n4  A281NPSIMI1C2R  B000052Y33  1052611200",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>item_id</th>\n      <th>time</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UX</td>\n      <td>1023840000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UM</td>\n      <td>1024185600</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B0000535UN</td>\n      <td>1024185600</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>AWIF8AR75LL9L</td>\n      <td>B000065DK4</td>\n      <td>1036627200</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>A281NPSIMI1C2R</td>\n      <td>B000052Y33</td>\n      <td>1052611200</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "import numpy as np\n",
    "\n",
    "min_u_num, min_i_num = 5, 5\n",
    "\n",
    "def get_illegal_ids_by_inter_num(df, field, max_num=None, min_num=None):\n",
    "    if field is None:\n",
    "        return set()\n",
    "    if max_num is None and min_num is None:\n",
    "        return set()\n",
    "\n",
    "    max_num = max_num or np.inf\n",
    "    min_num = min_num or -1\n",
    "\n",
    "    ids = df[field].values\n",
    "    inter_num = Counter(ids)\n",
    "    ids = {id_ for id_ in inter_num if inter_num[id_] < min_num or inter_num[id_] > max_num}\n",
    "    print(f'{len(ids)} illegal_ids_by_inter_num, field={field}')\n",
    "\n",
    "    return ids\n",
    "\n",
    "\n",
    "def filter_by_k_core(df):\n",
    "    while True:\n",
    "        ban_users = get_illegal_ids_by_inter_num(df, field=userID, max_num=None, min_num=min_u_num)\n",
    "        ban_items = get_illegal_ids_by_inter_num(df, field=itemID, max_num=None, min_num=min_i_num)\n",
    "        if len(ban_users) == 0 and len(ban_items) == 0:\n",
    "            return\n",
    "\n",
    "        dropped_inter = pd.Series(False, index=df.index)\n",
    "        if userID:\n",
    "            dropped_inter |= df[userID].isin(ban_users)\n",
    "        if itemID:\n",
    "            dropped_inter |= df[itemID].isin(ban_items)\n",
    "        print(f'{len(dropped_inter)} dropped interactions')\n",
    "        df.drop(df.index[dropped_inter], inplace=True)\n",
    "filter_by_k_core(ui_df)\n",
    "print(f'k-core shape: {ui_df.shape}')\n",
    "print(f'shape after k-core: {ui_df.shape}')\n",
    "ui_df[:5]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_users:22363\n",
      "n_items:12101\n",
      "iteractions:198502\n",
      "sparsity percent:99.93%\n"
     ]
    }
   ],
   "source": [
    "i_mapping_file='i_id_mapping.csv'\n",
    "u_mapping_file='u_id_mapping.csv'\n",
    "\n",
    "uni_users=sorted(pd.unique(ui_df[userID]))\n",
    "uni_items=sorted(pd.unique(ui_df[itemID]))\n",
    "n_users=len(uni_users)\n",
    "n_items=len(uni_items)\n",
    "print(f'n_users:{n_users}')\n",
    "print(f'n_items:{n_items}')\n",
    "print(f'iteractions:{ui_df.shape[0]}')\n",
    "print('sparsity percent:{:.2%}'.format(1-ui_df.shape[0]/(n_users*n_items)))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load mapping\n"
     ]
    }
   ],
   "source": [
    "#start from 0\n",
    "u_id_map={k:i for i,k in enumerate(uni_users)}\n",
    "i_id_map={k:i for i,k in enumerate(uni_items)}\n",
    "\n",
    "ui_df[userID]=ui_df[userID].map(u_id_map)\n",
    "ui_df[itemID]=ui_df[itemID].map(i_id_map)\n",
    "ui_df[userID]=ui_df[userID].astype(int)\n",
    "ui_df[itemID]=ui_df[itemID].astype(int)\n",
    "\n",
    "u_df = pd.DataFrame(list(u_id_map.items()), columns=['reviewerID', userID])\n",
    "i_df = pd.DataFrame(list(i_id_map.items()), columns=['asin', itemID])\n",
    "\n",
    "u_df.to_csv(os.path.join(RAW_PATH, u_mapping_file), sep='\\t', index=False)\n",
    "i_df.to_csv(os.path.join(RAW_PATH, i_mapping_file), sep='\\t', index=False)\n",
    "\n",
    "print('Load mapping')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "   user_id  item_id        time\n0     7187      103  1023840000\n1     7187      100  1024185600\n2     7187      101  1024185600\n3    21814      151  1036627200\n4     7187       22  1052611200",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>item_id</th>\n      <th>time</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7187</td>\n      <td>103</td>\n      <td>1023840000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7187</td>\n      <td>100</td>\n      <td>1024185600</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>7187</td>\n      <td>101</td>\n      <td>1024185600</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>21814</td>\n      <td>151</td>\n      <td>1036627200</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7187</td>\n      <td>22</td>\n      <td>1052611200</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ui_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "              reviewerID  user_id\n0  A00414041RD0BXM6WK0GX        0\n1  A00473363TJ8YSZ3YAGG9        1\n2  A00700212KB3K0MVESPIY        2\n3   A0078719IR14X3NNUG0F        3\n4  A01198201H0E3GHV2Z17I        4",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>reviewerID</th>\n      <th>user_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>A00414041RD0BXM6WK0GX</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>A00473363TJ8YSZ3YAGG9</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>A00700212KB3K0MVESPIY</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>A0078719IR14X3NNUG0F</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>A01198201H0E3GHV2Z17I</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "         asin  item_id\n0  7806397051        0\n1  9759091062        1\n2  9788072216        2\n3  9790790961        3\n4  9790794231        4",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>asin</th>\n      <th>item_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7806397051</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>9759091062</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>9788072216</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>9790790961</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9790794231</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "ui_df.to_csv(os.path.join(RAW_PATH, \"ui_interaction.csv\"), sep='\\t', index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### ui_df (user_id,item_id,time) sorted by time,\n",
    "### u_df (reviewerID,user_id) sorted by reviewerID,\n",
    "### i_df (asin,item_id) sorted by asin\n",
    "\n",
    "### #------------------Universal settings completed---------------------\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 2.reindex meta features (only item data)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import gzip\n",
    "\n",
    "\n",
    "os.chdir(\"/data/Chester/preprocessing\")\n",
    "os.getcwd()\n",
    "\n",
    "DATASET=\"Beauty\"\n",
    "RAW_PATH=os.path.join('./',DATASET)\n",
    "META_FILE=\"meta_{}.json.gz\".format(DATASET)\n",
    "i_id_mapping = 'i_id_mapping.csv'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i_id_mapping.csv (12101, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": "         asin  item_id\n0  7806397051        0\n1  9759091062        1\n2  9788072216        2\n3  9790790961        3\n4  9790794231        4",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>asin</th>\n      <th>item_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7806397051</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>9759091062</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>9788072216</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>9790790961</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9790794231</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#i_df is item mapping csv (asin,item_id)sorted by asin\n",
    "i_df=pd.read_csv(os.path.join(RAW_PATH,i_id_mapping),sep='\\t')\n",
    "print(f'{i_id_mapping} {i_df.shape}')\n",
    "i_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "item_meta_data:(259204, 9)\n"
     ]
    },
    {
     "data": {
      "text/plain": "         asin                                        description  \\\n0  0205616461  As we age, our once youthful, healthy skin suc...   \n1  0558925278  Mineral Powder Brush--Apply powder or mineral ...   \n2  0733001998  From the Greek island of Chios, this Mastiha b...   \n3  0737104473  Limited edition Hello Kitty Lipstick featuring...   \n4  0762451459  The mermaid is an elusive (okay, mythical) cre...   \n\n                                               title  \\\n0  Bio-Active Anti-Aging Serum (Firming Ultra-Hyd...   \n1  Eco Friendly Ecotools Quality Natural Bamboo C...   \n2                                Mastiha Body Lotion   \n3  Hello Kitty Lustre Lipstick (See sellers comme...   \n4        Stephanie Johnson Mermaid Round Snap Mirror   \n\n                                               imUrl  \\\n0  http://ecx.images-amazon.com/images/I/41DecrGO...   \n1  http://ecx.images-amazon.com/images/I/51L%2BzY...   \n2  http://ecx.images-amazon.com/images/I/311WK5y1...   \n3  http://ecx.images-amazon.com/images/I/31u6Hrzk...   \n4  http://ecx.images-amazon.com/images/I/41y2%2BF...   \n\n                            salesRank  \\\n0  {'Health & Personal Care': 461765}   \n1                  {'Beauty': 402875}   \n2                  {'Beauty': 540255}   \n3                  {'Beauty': 931125}   \n4                                 NaN   \n\n                                          categories  price related brand  \n0  [[Beauty, Skin Care, Face, Creams & Moisturize...    NaN     NaN   NaN  \n1  [[Beauty, Tools & Accessories, Makeup Brushes ...    NaN     NaN   NaN  \n2  [[Beauty, Skin Care, Body, Moisturizers, Lotio...    NaN     NaN   NaN  \n3                 [[Beauty, Makeup, Lips, Lipstick]]    NaN     NaN   NaN  \n4  [[Beauty, Tools & Accessories, Mirrors, Makeup...  19.98     NaN   NaN  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>asin</th>\n      <th>description</th>\n      <th>title</th>\n      <th>imUrl</th>\n      <th>salesRank</th>\n      <th>categories</th>\n      <th>price</th>\n      <th>related</th>\n      <th>brand</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0205616461</td>\n      <td>As we age, our once youthful, healthy skin suc...</td>\n      <td>Bio-Active Anti-Aging Serum (Firming Ultra-Hyd...</td>\n      <td>http://ecx.images-amazon.com/images/I/41DecrGO...</td>\n      <td>{'Health &amp; Personal Care': 461765}</td>\n      <td>[[Beauty, Skin Care, Face, Creams &amp; Moisturize...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0558925278</td>\n      <td>Mineral Powder Brush--Apply powder or mineral ...</td>\n      <td>Eco Friendly Ecotools Quality Natural Bamboo C...</td>\n      <td>http://ecx.images-amazon.com/images/I/51L%2BzY...</td>\n      <td>{'Beauty': 402875}</td>\n      <td>[[Beauty, Tools &amp; Accessories, Makeup Brushes ...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0733001998</td>\n      <td>From the Greek island of Chios, this Mastiha b...</td>\n      <td>Mastiha Body Lotion</td>\n      <td>http://ecx.images-amazon.com/images/I/311WK5y1...</td>\n      <td>{'Beauty': 540255}</td>\n      <td>[[Beauty, Skin Care, Body, Moisturizers, Lotio...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0737104473</td>\n      <td>Limited edition Hello Kitty Lipstick featuring...</td>\n      <td>Hello Kitty Lustre Lipstick (See sellers comme...</td>\n      <td>http://ecx.images-amazon.com/images/I/31u6Hrzk...</td>\n      <td>{'Beauty': 931125}</td>\n      <td>[[Beauty, Makeup, Lips, Lipstick]]</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0762451459</td>\n      <td>The mermaid is an elusive (okay, mythical) cre...</td>\n      <td>Stephanie Johnson Mermaid Round Snap Mirror</td>\n      <td>http://ecx.images-amazon.com/images/I/41y2%2BF...</td>\n      <td>NaN</td>\n      <td>[[Beauty, Tools &amp; Accessories, Mirrors, Makeup...</td>\n      <td>19.98</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        yield eval(l)\n",
    "\n",
    "def get_df(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')\n",
    "\n",
    "i_meta_df=get_df(os.path.join(RAW_PATH, META_FILE))\n",
    "\n",
    "print(f\"item_meta_data:{i_meta_df.shape}\")\n",
    "i_meta_df.head()\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "data": {
      "text/plain": "(259204, 10)"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#remapping\n",
    "map_dict=dict(zip(i_df['asin'],i_df[\"item_id\"]))\n",
    "\n",
    "i_meta_df['item_id']=i_meta_df['asin'].map(map_dict)\n",
    "i_meta_df.shape"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "data": {
      "text/plain": "(12101, 10)"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_meta_df.dropna(subset=['item_id'],inplace=True)\n",
    "i_meta_df.shape"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "data": {
      "text/plain": "         asin                                        description  \\\n0  7806397051  An extensive range of 15 multiple vibrant long...   \n1  9759091062  Xtreme Brite  Brightening gel is a highly conc...   \n2  9788072216  Prada Candy By Prada Eau De Parfum Spray 1.7 O...   \n3  9790790961  Versace Bright Crystal Perfume for Women 3 oz ...   \n4  9790794231  STELLA For Women By STELLA MCCARTNEY 1.7 oz ED...   \n\n                                               title  \\\n0  WAWO 15 Color Professionl Makeup Eyeshadow Cam...   \n1                  Xtreme Brite Brightening Gel 1oz.   \n2  Prada Candy By Prada Eau De Parfum Spray 1.7 O...   \n3  Versace Bright Crystal Eau de Toilette Spray f...   \n4                            Stella McCartney Stella   \n\n                                               imUrl           salesRank  \\\n0  http://ecx.images-amazon.com/images/I/41Rn18Oe...   {'Beauty': 10486}   \n1  http://ecx.images-amazon.com/images/I/41QWW9v1...   {'Beauty': 52254}   \n2  http://ecx.images-amazon.com/images/I/51iT2k6L...   {'Beauty': 78916}   \n3  http://ecx.images-amazon.com/images/I/418LYGLE...     {'Beauty': 764}   \n4  http://ecx.images-amazon.com/images/I/31L2n60J...  {'Beauty': 142503}   \n\n                                          categories  price  \\\n0  [[Beauty, Makeup, Face, Concealers & Neutraliz...   5.04   \n1  [[Beauty, Hair Care, Styling Products, Creams,...  19.99   \n2      [[Beauty, Fragrance, Women's, Eau de Parfum]]  65.86   \n3    [[Beauty, Fragrance, Women's, Eau de Toilette]]  52.33   \n4      [[Beauty, Fragrance, Women's, Eau de Parfum]]    NaN   \n\n                                             related         brand  item_id  \n0  {'also_bought': ['B00KR26VFE', 'B00E7LQHZ0', '...          COKA        0  \n1  {'also_bought': ['B0054GLD1U', 'B003BRZCUC', '...  Xtreme Brite        1  \n2  {'also_bought': ['B006C5OHSI', 'B006P14842', '...         Prada        2  \n3  {'also_bought': ['B007P7OPQQ', 'B0017JT658', '...       Versace        3  \n4  {'also_bought': ['B0019M21OQ', 'B000E7YM8K', '...           NaN        4  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>asin</th>\n      <th>description</th>\n      <th>title</th>\n      <th>imUrl</th>\n      <th>salesRank</th>\n      <th>categories</th>\n      <th>price</th>\n      <th>related</th>\n      <th>brand</th>\n      <th>item_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7806397051</td>\n      <td>An extensive range of 15 multiple vibrant long...</td>\n      <td>WAWO 15 Color Professionl Makeup Eyeshadow Cam...</td>\n      <td>http://ecx.images-amazon.com/images/I/41Rn18Oe...</td>\n      <td>{'Beauty': 10486}</td>\n      <td>[[Beauty, Makeup, Face, Concealers &amp; Neutraliz...</td>\n      <td>5.04</td>\n      <td>{'also_bought': ['B00KR26VFE', 'B00E7LQHZ0', '...</td>\n      <td>COKA</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>9759091062</td>\n      <td>Xtreme Brite  Brightening gel is a highly conc...</td>\n      <td>Xtreme Brite Brightening Gel 1oz.</td>\n      <td>http://ecx.images-amazon.com/images/I/41QWW9v1...</td>\n      <td>{'Beauty': 52254}</td>\n      <td>[[Beauty, Hair Care, Styling Products, Creams,...</td>\n      <td>19.99</td>\n      <td>{'also_bought': ['B0054GLD1U', 'B003BRZCUC', '...</td>\n      <td>Xtreme Brite</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>9788072216</td>\n      <td>Prada Candy By Prada Eau De Parfum Spray 1.7 O...</td>\n      <td>Prada Candy By Prada Eau De Parfum Spray 1.7 O...</td>\n      <td>http://ecx.images-amazon.com/images/I/51iT2k6L...</td>\n      <td>{'Beauty': 78916}</td>\n      <td>[[Beauty, Fragrance, Women's, Eau de Parfum]]</td>\n      <td>65.86</td>\n      <td>{'also_bought': ['B006C5OHSI', 'B006P14842', '...</td>\n      <td>Prada</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>9790790961</td>\n      <td>Versace Bright Crystal Perfume for Women 3 oz ...</td>\n      <td>Versace Bright Crystal Eau de Toilette Spray f...</td>\n      <td>http://ecx.images-amazon.com/images/I/418LYGLE...</td>\n      <td>{'Beauty': 764}</td>\n      <td>[[Beauty, Fragrance, Women's, Eau de Toilette]]</td>\n      <td>52.33</td>\n      <td>{'also_bought': ['B007P7OPQQ', 'B0017JT658', '...</td>\n      <td>Versace</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9790794231</td>\n      <td>STELLA For Women By STELLA MCCARTNEY 1.7 oz ED...</td>\n      <td>Stella McCartney Stella</td>\n      <td>http://ecx.images-amazon.com/images/I/31L2n60J...</td>\n      <td>{'Beauty': 142503}</td>\n      <td>[[Beauty, Fragrance, Women's, Eau de Parfum]]</td>\n      <td>NaN</td>\n      <td>{'also_bought': ['B0019M21OQ', 'B000E7YM8K', '...</td>\n      <td>NaN</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_meta_df['item_id']=i_meta_df['item_id'].astype(int)\n",
    "i_meta_df=i_meta_df.sort_values(by=['item_id'],kind='mergesort').reset_index(drop=True)#very important\n",
    "i_meta_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "data": {
      "text/plain": "['item_id',\n 'asin',\n 'description',\n 'title',\n 'imUrl',\n 'salesRank',\n 'categories',\n 'price',\n 'related',\n 'brand']"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "origin_cols=i_meta_df.columns.tolist()\n",
    "\n",
    "target_cols=[origin_cols[-1]]+origin_cols[:-1]\n",
    "target_cols"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "   item_id        asin                                        description  \\\n0        0  7806397051  An extensive range of 15 multiple vibrant long...   \n1        1  9759091062  Xtreme Brite  Brightening gel is a highly conc...   \n2        2  9788072216  Prada Candy By Prada Eau De Parfum Spray 1.7 O...   \n3        3  9790790961  Versace Bright Crystal Perfume for Women 3 oz ...   \n4        4  9790794231  STELLA For Women By STELLA MCCARTNEY 1.7 oz ED...   \n\n                                               title  \\\n0  WAWO 15 Color Professionl Makeup Eyeshadow Cam...   \n1                  Xtreme Brite Brightening Gel 1oz.   \n2  Prada Candy By Prada Eau De Parfum Spray 1.7 O...   \n3  Versace Bright Crystal Eau de Toilette Spray f...   \n4                            Stella McCartney Stella   \n\n                                               imUrl           salesRank  \\\n0  http://ecx.images-amazon.com/images/I/41Rn18Oe...   {'Beauty': 10486}   \n1  http://ecx.images-amazon.com/images/I/41QWW9v1...   {'Beauty': 52254}   \n2  http://ecx.images-amazon.com/images/I/51iT2k6L...   {'Beauty': 78916}   \n3  http://ecx.images-amazon.com/images/I/418LYGLE...     {'Beauty': 764}   \n4  http://ecx.images-amazon.com/images/I/31L2n60J...  {'Beauty': 142503}   \n\n                                          categories  price  \\\n0  [[Beauty, Makeup, Face, Concealers & Neutraliz...   5.04   \n1  [[Beauty, Hair Care, Styling Products, Creams,...  19.99   \n2      [[Beauty, Fragrance, Women's, Eau de Parfum]]  65.86   \n3    [[Beauty, Fragrance, Women's, Eau de Toilette]]  52.33   \n4      [[Beauty, Fragrance, Women's, Eau de Parfum]]    NaN   \n\n                                             related         brand  \n0  {'also_bought': ['B00KR26VFE', 'B00E7LQHZ0', '...          COKA  \n1  {'also_bought': ['B0054GLD1U', 'B003BRZCUC', '...  Xtreme Brite  \n2  {'also_bought': ['B006C5OHSI', 'B006P14842', '...         Prada  \n3  {'also_bought': ['B007P7OPQQ', 'B0017JT658', '...       Versace  \n4  {'also_bought': ['B0019M21OQ', 'B000E7YM8K', '...           NaN  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>item_id</th>\n      <th>asin</th>\n      <th>description</th>\n      <th>title</th>\n      <th>imUrl</th>\n      <th>salesRank</th>\n      <th>categories</th>\n      <th>price</th>\n      <th>related</th>\n      <th>brand</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>7806397051</td>\n      <td>An extensive range of 15 multiple vibrant long...</td>\n      <td>WAWO 15 Color Professionl Makeup Eyeshadow Cam...</td>\n      <td>http://ecx.images-amazon.com/images/I/41Rn18Oe...</td>\n      <td>{'Beauty': 10486}</td>\n      <td>[[Beauty, Makeup, Face, Concealers &amp; Neutraliz...</td>\n      <td>5.04</td>\n      <td>{'also_bought': ['B00KR26VFE', 'B00E7LQHZ0', '...</td>\n      <td>COKA</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>9759091062</td>\n      <td>Xtreme Brite  Brightening gel is a highly conc...</td>\n      <td>Xtreme Brite Brightening Gel 1oz.</td>\n      <td>http://ecx.images-amazon.com/images/I/41QWW9v1...</td>\n      <td>{'Beauty': 52254}</td>\n      <td>[[Beauty, Hair Care, Styling Products, Creams,...</td>\n      <td>19.99</td>\n      <td>{'also_bought': ['B0054GLD1U', 'B003BRZCUC', '...</td>\n      <td>Xtreme Brite</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>9788072216</td>\n      <td>Prada Candy By Prada Eau De Parfum Spray 1.7 O...</td>\n      <td>Prada Candy By Prada Eau De Parfum Spray 1.7 O...</td>\n      <td>http://ecx.images-amazon.com/images/I/51iT2k6L...</td>\n      <td>{'Beauty': 78916}</td>\n      <td>[[Beauty, Fragrance, Women's, Eau de Parfum]]</td>\n      <td>65.86</td>\n      <td>{'also_bought': ['B006C5OHSI', 'B006P14842', '...</td>\n      <td>Prada</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>9790790961</td>\n      <td>Versace Bright Crystal Perfume for Women 3 oz ...</td>\n      <td>Versace Bright Crystal Eau de Toilette Spray f...</td>\n      <td>http://ecx.images-amazon.com/images/I/418LYGLE...</td>\n      <td>{'Beauty': 764}</td>\n      <td>[[Beauty, Fragrance, Women's, Eau de Toilette]]</td>\n      <td>52.33</td>\n      <td>{'also_bought': ['B007P7OPQQ', 'B0017JT658', '...</td>\n      <td>Versace</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>9790794231</td>\n      <td>STELLA For Women By STELLA MCCARTNEY 1.7 oz ED...</td>\n      <td>Stella McCartney Stella</td>\n      <td>http://ecx.images-amazon.com/images/I/31L2n60J...</td>\n      <td>{'Beauty': 142503}</td>\n      <td>[[Beauty, Fragrance, Women's, Eau de Parfum]]</td>\n      <td>NaN</td>\n      <td>{'also_bought': ['B0019M21OQ', 'B000E7YM8K', '...</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_i_df=i_meta_df[target_cols]\n",
    "target_i_df.to_csv(os.path.join(RAW_PATH,'i_meta_{}.csv'.format(DATASET)),index=False)\n",
    "target_i_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique items:12101\n",
      "min/max of unique items:0,12100\n"
     ]
    }
   ],
   "source": [
    "uni_items=target_i_df['item_id'].unique()\n",
    "print(f'unique items:{len(uni_items)}')\n",
    "print(f'min/max of unique items:{min(uni_items)},{max(uni_items)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 3.feature encoder"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/Chester/preprocessing\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import torch\n",
    "\n",
    "seed=123\n",
    "\n",
    "random.seed(seed)\n",
    "os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = True\n",
    "\n",
    "os.chdir(\"/data/Chester/preprocessing\")\n",
    "print(os.getcwd())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "data": {
      "text/plain": "(12101, 10)"
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATASET=\"Beauty\"\n",
    "RAW_PATH=os.path.join('./',DATASET)\n",
    "i_meta_df_name='i_meta_{}.csv'.format(DATASET)\n",
    "\n",
    "item_meta_file=os.path.join(RAW_PATH,i_meta_df_name)\n",
    "i_meta_df=pd.read_csv(item_meta_file)\n",
    "i_meta_df.shape"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "data": {
      "text/plain": "   item_id        asin                                        description  \\\n0        0  7806397051  An extensive range of 15 multiple vibrant long...   \n1        1  9759091062  Xtreme Brite  Brightening gel is a highly conc...   \n2        2  9788072216  Prada Candy By Prada Eau De Parfum Spray 1.7 O...   \n3        3  9790790961  Versace Bright Crystal Perfume for Women 3 oz ...   \n4        4  9790794231  STELLA For Women By STELLA MCCARTNEY 1.7 oz ED...   \n\n                                               title  \\\n0  WAWO 15 Color Professionl Makeup Eyeshadow Cam...   \n1                  Xtreme Brite Brightening Gel 1oz.   \n2  Prada Candy By Prada Eau De Parfum Spray 1.7 O...   \n3  Versace Bright Crystal Eau de Toilette Spray f...   \n4                            Stella McCartney Stella   \n\n                                               imUrl           salesRank  \\\n0  http://ecx.images-amazon.com/images/I/41Rn18Oe...   {'Beauty': 10486}   \n1  http://ecx.images-amazon.com/images/I/41QWW9v1...   {'Beauty': 52254}   \n2  http://ecx.images-amazon.com/images/I/51iT2k6L...   {'Beauty': 78916}   \n3  http://ecx.images-amazon.com/images/I/418LYGLE...     {'Beauty': 764}   \n4  http://ecx.images-amazon.com/images/I/31L2n60J...  {'Beauty': 142503}   \n\n                                          categories  price  \\\n0  [['Beauty', 'Makeup', 'Face', 'Concealers & Ne...   5.04   \n1  [['Beauty', 'Hair Care', 'Styling Products', '...  19.99   \n2  [['Beauty', 'Fragrance', \"Women's\", 'Eau de Pa...  65.86   \n3  [['Beauty', 'Fragrance', \"Women's\", 'Eau de To...  52.33   \n4  [['Beauty', 'Fragrance', \"Women's\", 'Eau de Pa...    NaN   \n\n                                             related         brand  \n0  {'also_bought': ['B00KR26VFE', 'B00E7LQHZ0', '...          COKA  \n1  {'also_bought': ['B0054GLD1U', 'B003BRZCUC', '...  Xtreme Brite  \n2  {'also_bought': ['B006C5OHSI', 'B006P14842', '...         Prada  \n3  {'also_bought': ['B007P7OPQQ', 'B0017JT658', '...       Versace  \n4  {'also_bought': ['B0019M21OQ', 'B000E7YM8K', '...           NaN  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>item_id</th>\n      <th>asin</th>\n      <th>description</th>\n      <th>title</th>\n      <th>imUrl</th>\n      <th>salesRank</th>\n      <th>categories</th>\n      <th>price</th>\n      <th>related</th>\n      <th>brand</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>7806397051</td>\n      <td>An extensive range of 15 multiple vibrant long...</td>\n      <td>WAWO 15 Color Professionl Makeup Eyeshadow Cam...</td>\n      <td>http://ecx.images-amazon.com/images/I/41Rn18Oe...</td>\n      <td>{'Beauty': 10486}</td>\n      <td>[['Beauty', 'Makeup', 'Face', 'Concealers &amp; Ne...</td>\n      <td>5.04</td>\n      <td>{'also_bought': ['B00KR26VFE', 'B00E7LQHZ0', '...</td>\n      <td>COKA</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>9759091062</td>\n      <td>Xtreme Brite  Brightening gel is a highly conc...</td>\n      <td>Xtreme Brite Brightening Gel 1oz.</td>\n      <td>http://ecx.images-amazon.com/images/I/41QWW9v1...</td>\n      <td>{'Beauty': 52254}</td>\n      <td>[['Beauty', 'Hair Care', 'Styling Products', '...</td>\n      <td>19.99</td>\n      <td>{'also_bought': ['B0054GLD1U', 'B003BRZCUC', '...</td>\n      <td>Xtreme Brite</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>9788072216</td>\n      <td>Prada Candy By Prada Eau De Parfum Spray 1.7 O...</td>\n      <td>Prada Candy By Prada Eau De Parfum Spray 1.7 O...</td>\n      <td>http://ecx.images-amazon.com/images/I/51iT2k6L...</td>\n      <td>{'Beauty': 78916}</td>\n      <td>[['Beauty', 'Fragrance', \"Women's\", 'Eau de Pa...</td>\n      <td>65.86</td>\n      <td>{'also_bought': ['B006C5OHSI', 'B006P14842', '...</td>\n      <td>Prada</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>9790790961</td>\n      <td>Versace Bright Crystal Perfume for Women 3 oz ...</td>\n      <td>Versace Bright Crystal Eau de Toilette Spray f...</td>\n      <td>http://ecx.images-amazon.com/images/I/418LYGLE...</td>\n      <td>{'Beauty': 764}</td>\n      <td>[['Beauty', 'Fragrance', \"Women's\", 'Eau de To...</td>\n      <td>52.33</td>\n      <td>{'also_bought': ['B007P7OPQQ', 'B0017JT658', '...</td>\n      <td>Versace</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>9790794231</td>\n      <td>STELLA For Women By STELLA MCCARTNEY 1.7 oz ED...</td>\n      <td>Stella McCartney Stella</td>\n      <td>http://ecx.images-amazon.com/images/I/31L2n60J...</td>\n      <td>{'Beauty': 142503}</td>\n      <td>[['Beauty', 'Fragrance', \"Women's\", 'Eau de Pa...</td>\n      <td>NaN</td>\n      <td>{'also_bought': ['B0019M21OQ', 'B000E7YM8K', '...</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_meta_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "title null: (7, 10)\n",
      "price null: (585, 10)\n",
      "imUrl null: (7, 10)\n",
      "brand null: (2098, 10)\n",
      "categories null: (0, 10)\n",
      "description null: (939, 10)\n"
     ]
    }
   ],
   "source": [
    "#sentences:title+brand+category+description | ALL have title+description\n",
    "title_na_df=i_meta_df[i_meta_df['title'].isnull()]\n",
    "print(\"title null:\",title_na_df.shape)\n",
    "\n",
    "price_na_df=i_meta_df[i_meta_df['price'].isnull()]\n",
    "print(\"price null:\",price_na_df.shape)\n",
    "\n",
    "imUrl_na_df=i_meta_df[i_meta_df['imUrl'].isnull()]\n",
    "print(\"imUrl null:\",imUrl_na_df.shape)\n",
    "\n",
    "brand_na_df=i_meta_df[i_meta_df['brand'].isnull()]\n",
    "print('brand null:',brand_na_df.shape)\n",
    "\n",
    "categories_na_df=i_meta_df[i_meta_df['categories'].isnull()]\n",
    "print(\"categories null:\",categories_na_df.shape)\n",
    "\n",
    "description_na_df=i_meta_df[i_meta_df['description'].isnull()]\n",
    "print(\"description null:\",description_na_df.shape)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Text feature processing"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "i_meta_df['title']=i_meta_df['title'].fillna(\" \")\n",
    "i_meta_df['brand']=i_meta_df['brand'].fillna(\" \")\n",
    "i_meta_df['description']=i_meta_df['description'].fillna(\" \")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12101 12101\n",
      "max_sentences_len:1254\n",
      "min_sentences_len:8\n",
      "avg_sentences_len:88.5107015949095\n"
     ]
    }
   ],
   "source": [
    "#-------------Text Features------------------\n",
    "#remove part html:\n",
    "import re\n",
    "\n",
    "#---------------re remove html---------------------\n",
    "pattern = re.compile(r'<.*?>',re.S)\n",
    "\n",
    "clean_sentences=[]\n",
    "max_sentences_len=0\n",
    "min_sentences_len=9999\n",
    "sum_sentences_len=0\n",
    "sentences_n=0\n",
    "\n",
    "for i,row in i_meta_df.iterrows():\n",
    "    sen=row['title']+\" \"+row['brand']+\" \"\n",
    "    cates=eval(row['categories'])\n",
    "    if isinstance(cates,list):\n",
    "        for c in cates[0]:\n",
    "            sen=sen+c+\" \"\n",
    "    sen+=row[\"description\"]\n",
    "    #----------remove html-------------\n",
    "    sen = pattern.sub('', sen)\n",
    "    #----------------------------------\n",
    "    sen=sen.replace('\\n',\" \")\n",
    "\n",
    "    sen_len=len(sen.split(\" \"))\n",
    "    max_sentences_len= sen_len if sen_len>max_sentences_len else max_sentences_len\n",
    "    min_sentences_len=sen_len if sen_len<min_sentences_len else min_sentences_len\n",
    "    sum_sentences_len+=sen_len\n",
    "\n",
    "    sentences_n+=1\n",
    "    clean_sentences.append(sen)\n",
    "\n",
    "print(len(clean_sentences),sentences_n)\n",
    "print(f\"max_sentences_len:{max_sentences_len}\")\n",
    "print(f\"min_sentences_len:{min_sentences_len}\")\n",
    "print(f\"avg_sentences_len:{sum_sentences_len/sentences_n}\")\n",
    "#sum sentences avg.len to 77 clip limited,so use bert"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda-11.2/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
      "CUDA SETUP: Detected CUDA version 112\n",
      "CUDA SETUP: Loading binary /home/Chester/.conda/envs/graph_llm/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda112.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/Chester/.conda/envs/graph_llm/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /home/server/anaconda3 did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\n"
     ]
    },
    {
     "data": {
      "text/plain": "BertModel(\n  (embeddings): BertEmbeddings(\n    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n    (position_embeddings): Embedding(512, 768)\n    (token_type_embeddings): Embedding(2, 768)\n    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n    (dropout): Dropout(p=0.1, inplace=False)\n  )\n  (encoder): BertEncoder(\n    (layer): ModuleList(\n      (0-11): 12 x BertLayer(\n        (attention): BertAttention(\n          (self): BertSelfAttention(\n            (query): Linear(in_features=768, out_features=768, bias=True)\n            (key): Linear(in_features=768, out_features=768, bias=True)\n            (value): Linear(in_features=768, out_features=768, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n          (output): BertSelfOutput(\n            (dense): Linear(in_features=768, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (intermediate): BertIntermediate(\n          (dense): Linear(in_features=768, out_features=3072, bias=True)\n          (intermediate_act_fn): GELUActivation()\n        )\n        (output): BertOutput(\n          (dense): Linear(in_features=3072, out_features=768, bias=True)\n          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (pooler): BertPooler(\n    (dense): Linear(in_features=768, out_features=768, bias=True)\n    (activation): Tanh()\n  )\n)"
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Hugging face bert\n",
    "from transformers import BertModel,BertTokenizer\n",
    "import torch\n",
    "import os\n",
    "\n",
    "model_path=os.path.join(os.getcwd(),'bert-base-uncased')\n",
    "tokenizer = BertTokenizer.from_pretrained(model_path)\n",
    "model=BertModel.from_pretrained(model_path)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "data": {
      "text/plain": "device(type='cuda', index=0)"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.device"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0-32\n",
      "torch.Size([32, 768])\n",
      "32-64\n",
      "torch.Size([32, 768])\n",
      "64-96\n",
      "torch.Size([32, 768])\n",
      "96-128\n",
      "torch.Size([32, 768])\n",
      "128-160\n",
      "torch.Size([32, 768])\n",
      "160-192\n",
      "torch.Size([32, 768])\n",
      "192-224\n",
      "torch.Size([32, 768])\n",
      "224-256\n",
      "torch.Size([32, 768])\n",
      "256-288\n",
      "torch.Size([32, 768])\n",
      "288-320\n",
      "torch.Size([32, 768])\n",
      "320-352\n",
      "torch.Size([32, 768])\n",
      "352-384\n",
      "torch.Size([32, 768])\n",
      "384-416\n",
      "torch.Size([32, 768])\n",
      "416-448\n",
      "torch.Size([32, 768])\n",
      "448-480\n",
      "torch.Size([32, 768])\n",
      "480-512\n",
      "torch.Size([32, 768])\n",
      "512-544\n",
      "torch.Size([32, 768])\n",
      "544-576\n",
      "torch.Size([32, 768])\n",
      "576-608\n",
      "torch.Size([32, 768])\n",
      "608-640\n",
      "torch.Size([32, 768])\n",
      "640-672\n",
      "torch.Size([32, 768])\n",
      "672-704\n",
      "torch.Size([32, 768])\n",
      "704-736\n",
      "torch.Size([32, 768])\n",
      "736-768\n",
      "torch.Size([32, 768])\n",
      "768-800\n",
      "torch.Size([32, 768])\n",
      "800-832\n",
      "torch.Size([32, 768])\n",
      "832-864\n",
      "torch.Size([32, 768])\n",
      "864-896\n",
      "torch.Size([32, 768])\n",
      "896-928\n",
      "torch.Size([32, 768])\n",
      "928-960\n",
      "torch.Size([32, 768])\n",
      "960-992\n",
      "torch.Size([32, 768])\n",
      "992-1024\n",
      "torch.Size([32, 768])\n",
      "1024-1056\n",
      "torch.Size([32, 768])\n",
      "1056-1088\n",
      "torch.Size([32, 768])\n",
      "1088-1120\n",
      "torch.Size([32, 768])\n",
      "1120-1152\n",
      "torch.Size([32, 768])\n",
      "1152-1184\n",
      "torch.Size([32, 768])\n",
      "1184-1216\n",
      "torch.Size([32, 768])\n",
      "1216-1248\n",
      "torch.Size([32, 768])\n",
      "1248-1280\n",
      "torch.Size([32, 768])\n",
      "1280-1312\n",
      "torch.Size([32, 768])\n",
      "1312-1344\n",
      "torch.Size([32, 768])\n",
      "1344-1376\n",
      "torch.Size([32, 768])\n",
      "1376-1408\n",
      "torch.Size([32, 768])\n",
      "1408-1440\n",
      "torch.Size([32, 768])\n",
      "1440-1472\n",
      "torch.Size([32, 768])\n",
      "1472-1504\n",
      "torch.Size([32, 768])\n",
      "1504-1536\n",
      "torch.Size([32, 768])\n",
      "1536-1568\n",
      "torch.Size([32, 768])\n",
      "1568-1600\n",
      "torch.Size([32, 768])\n",
      "1600-1632\n",
      "torch.Size([32, 768])\n",
      "1632-1664\n",
      "torch.Size([32, 768])\n",
      "1664-1696\n",
      "torch.Size([32, 768])\n",
      "1696-1728\n",
      "torch.Size([32, 768])\n",
      "1728-1760\n",
      "torch.Size([32, 768])\n",
      "1760-1792\n",
      "torch.Size([32, 768])\n",
      "1792-1824\n",
      "torch.Size([32, 768])\n",
      "1824-1856\n",
      "torch.Size([32, 768])\n",
      "1856-1888\n",
      "torch.Size([32, 768])\n",
      "1888-1920\n",
      "torch.Size([32, 768])\n",
      "1920-1952\n",
      "torch.Size([32, 768])\n",
      "1952-1984\n",
      "torch.Size([32, 768])\n",
      "1984-2016\n",
      "torch.Size([32, 768])\n",
      "2016-2048\n",
      "torch.Size([32, 768])\n",
      "2048-2080\n",
      "torch.Size([32, 768])\n",
      "2080-2112\n",
      "torch.Size([32, 768])\n",
      "2112-2144\n",
      "torch.Size([32, 768])\n",
      "2144-2176\n",
      "torch.Size([32, 768])\n",
      "2176-2208\n",
      "torch.Size([32, 768])\n",
      "2208-2240\n",
      "torch.Size([32, 768])\n",
      "2240-2272\n",
      "torch.Size([32, 768])\n",
      "2272-2304\n",
      "torch.Size([32, 768])\n",
      "2304-2336\n",
      "torch.Size([32, 768])\n",
      "2336-2368\n",
      "torch.Size([32, 768])\n",
      "2368-2400\n",
      "torch.Size([32, 768])\n",
      "2400-2432\n",
      "torch.Size([32, 768])\n",
      "2432-2464\n",
      "torch.Size([32, 768])\n",
      "2464-2496\n",
      "torch.Size([32, 768])\n",
      "2496-2528\n",
      "torch.Size([32, 768])\n",
      "2528-2560\n",
      "torch.Size([32, 768])\n",
      "2560-2592\n",
      "torch.Size([32, 768])\n",
      "2592-2624\n",
      "torch.Size([32, 768])\n",
      "2624-2656\n",
      "torch.Size([32, 768])\n",
      "2656-2688\n",
      "torch.Size([32, 768])\n",
      "2688-2720\n",
      "torch.Size([32, 768])\n",
      "2720-2752\n",
      "torch.Size([32, 768])\n",
      "2752-2784\n",
      "torch.Size([32, 768])\n",
      "2784-2816\n",
      "torch.Size([32, 768])\n",
      "2816-2848\n",
      "torch.Size([32, 768])\n",
      "2848-2880\n",
      "torch.Size([32, 768])\n",
      "2880-2912\n",
      "torch.Size([32, 768])\n",
      "2912-2944\n",
      "torch.Size([32, 768])\n",
      "2944-2976\n",
      "torch.Size([32, 768])\n",
      "2976-3008\n",
      "torch.Size([32, 768])\n",
      "3008-3040\n",
      "torch.Size([32, 768])\n",
      "3040-3072\n",
      "torch.Size([32, 768])\n",
      "3072-3104\n",
      "torch.Size([32, 768])\n",
      "3104-3136\n",
      "torch.Size([32, 768])\n",
      "3136-3168\n",
      "torch.Size([32, 768])\n",
      "3168-3200\n",
      "torch.Size([32, 768])\n",
      "3200-3232\n",
      "torch.Size([32, 768])\n",
      "3232-3264\n",
      "torch.Size([32, 768])\n",
      "3264-3296\n",
      "torch.Size([32, 768])\n",
      "3296-3328\n",
      "torch.Size([32, 768])\n",
      "3328-3360\n",
      "torch.Size([32, 768])\n",
      "3360-3392\n",
      "torch.Size([32, 768])\n",
      "3392-3424\n",
      "torch.Size([32, 768])\n",
      "3424-3456\n",
      "torch.Size([32, 768])\n",
      "3456-3488\n",
      "torch.Size([32, 768])\n",
      "3488-3520\n",
      "torch.Size([32, 768])\n",
      "3520-3552\n",
      "torch.Size([32, 768])\n",
      "3552-3584\n",
      "torch.Size([32, 768])\n",
      "3584-3616\n",
      "torch.Size([32, 768])\n",
      "3616-3648\n",
      "torch.Size([32, 768])\n",
      "3648-3680\n",
      "torch.Size([32, 768])\n",
      "3680-3712\n",
      "torch.Size([32, 768])\n",
      "3712-3744\n",
      "torch.Size([32, 768])\n",
      "3744-3776\n",
      "torch.Size([32, 768])\n",
      "3776-3808\n",
      "torch.Size([32, 768])\n",
      "3808-3840\n",
      "torch.Size([32, 768])\n",
      "3840-3872\n",
      "torch.Size([32, 768])\n",
      "3872-3904\n",
      "torch.Size([32, 768])\n",
      "3904-3936\n",
      "torch.Size([32, 768])\n",
      "3936-3968\n",
      "torch.Size([32, 768])\n",
      "3968-4000\n",
      "torch.Size([32, 768])\n",
      "4000-4032\n",
      "torch.Size([32, 768])\n",
      "4032-4064\n",
      "torch.Size([32, 768])\n",
      "4064-4096\n",
      "torch.Size([32, 768])\n",
      "4096-4128\n",
      "torch.Size([32, 768])\n",
      "4128-4160\n",
      "torch.Size([32, 768])\n",
      "4160-4192\n",
      "torch.Size([32, 768])\n",
      "4192-4224\n",
      "torch.Size([32, 768])\n",
      "4224-4256\n",
      "torch.Size([32, 768])\n",
      "4256-4288\n",
      "torch.Size([32, 768])\n",
      "4288-4320\n",
      "torch.Size([32, 768])\n",
      "4320-4352\n",
      "torch.Size([32, 768])\n",
      "4352-4384\n",
      "torch.Size([32, 768])\n",
      "4384-4416\n",
      "torch.Size([32, 768])\n",
      "4416-4448\n",
      "torch.Size([32, 768])\n",
      "4448-4480\n",
      "torch.Size([32, 768])\n",
      "4480-4512\n",
      "torch.Size([32, 768])\n",
      "4512-4544\n",
      "torch.Size([32, 768])\n",
      "4544-4576\n",
      "torch.Size([32, 768])\n",
      "4576-4608\n",
      "torch.Size([32, 768])\n",
      "4608-4640\n",
      "torch.Size([32, 768])\n",
      "4640-4672\n",
      "torch.Size([32, 768])\n",
      "4672-4704\n",
      "torch.Size([32, 768])\n",
      "4704-4736\n",
      "torch.Size([32, 768])\n",
      "4736-4768\n",
      "torch.Size([32, 768])\n",
      "4768-4800\n",
      "torch.Size([32, 768])\n",
      "4800-4832\n",
      "torch.Size([32, 768])\n",
      "4832-4864\n",
      "torch.Size([32, 768])\n",
      "4864-4896\n",
      "torch.Size([32, 768])\n",
      "4896-4928\n",
      "torch.Size([32, 768])\n",
      "4928-4960\n",
      "torch.Size([32, 768])\n",
      "4960-4992\n",
      "torch.Size([32, 768])\n",
      "4992-5024\n",
      "torch.Size([32, 768])\n",
      "5024-5056\n",
      "torch.Size([32, 768])\n",
      "5056-5088\n",
      "torch.Size([32, 768])\n",
      "5088-5120\n",
      "torch.Size([32, 768])\n",
      "5120-5152\n",
      "torch.Size([32, 768])\n",
      "5152-5184\n",
      "torch.Size([32, 768])\n",
      "5184-5216\n",
      "torch.Size([32, 768])\n",
      "5216-5248\n",
      "torch.Size([32, 768])\n",
      "5248-5280\n",
      "torch.Size([32, 768])\n",
      "5280-5312\n",
      "torch.Size([32, 768])\n",
      "5312-5344\n",
      "torch.Size([32, 768])\n",
      "5344-5376\n",
      "torch.Size([32, 768])\n",
      "5376-5408\n",
      "torch.Size([32, 768])\n",
      "5408-5440\n",
      "torch.Size([32, 768])\n",
      "5440-5472\n",
      "torch.Size([32, 768])\n",
      "5472-5504\n",
      "torch.Size([32, 768])\n",
      "5504-5536\n",
      "torch.Size([32, 768])\n",
      "5536-5568\n",
      "torch.Size([32, 768])\n",
      "5568-5600\n",
      "torch.Size([32, 768])\n",
      "5600-5632\n",
      "torch.Size([32, 768])\n",
      "5632-5664\n",
      "torch.Size([32, 768])\n",
      "5664-5696\n",
      "torch.Size([32, 768])\n",
      "5696-5728\n",
      "torch.Size([32, 768])\n",
      "5728-5760\n",
      "torch.Size([32, 768])\n",
      "5760-5792\n",
      "torch.Size([32, 768])\n",
      "5792-5824\n",
      "torch.Size([32, 768])\n",
      "5824-5856\n",
      "torch.Size([32, 768])\n",
      "5856-5888\n",
      "torch.Size([32, 768])\n",
      "5888-5920\n",
      "torch.Size([32, 768])\n",
      "5920-5952\n",
      "torch.Size([32, 768])\n",
      "5952-5984\n",
      "torch.Size([32, 768])\n",
      "5984-6016\n",
      "torch.Size([32, 768])\n",
      "6016-6048\n",
      "torch.Size([32, 768])\n",
      "6048-6080\n",
      "torch.Size([32, 768])\n",
      "6080-6112\n",
      "torch.Size([32, 768])\n",
      "6112-6144\n",
      "torch.Size([32, 768])\n",
      "6144-6176\n",
      "torch.Size([32, 768])\n",
      "6176-6208\n",
      "torch.Size([32, 768])\n",
      "6208-6240\n",
      "torch.Size([32, 768])\n",
      "6240-6272\n",
      "torch.Size([32, 768])\n",
      "6272-6304\n",
      "torch.Size([32, 768])\n",
      "6304-6336\n",
      "torch.Size([32, 768])\n",
      "6336-6368\n",
      "torch.Size([32, 768])\n",
      "6368-6400\n",
      "torch.Size([32, 768])\n",
      "6400-6432\n",
      "torch.Size([32, 768])\n",
      "6432-6464\n",
      "torch.Size([32, 768])\n",
      "6464-6496\n",
      "torch.Size([32, 768])\n",
      "6496-6528\n",
      "torch.Size([32, 768])\n",
      "6528-6560\n",
      "torch.Size([32, 768])\n",
      "6560-6592\n",
      "torch.Size([32, 768])\n",
      "6592-6624\n",
      "torch.Size([32, 768])\n",
      "6624-6656\n",
      "torch.Size([32, 768])\n",
      "6656-6688\n",
      "torch.Size([32, 768])\n",
      "6688-6720\n",
      "torch.Size([32, 768])\n",
      "6720-6752\n",
      "torch.Size([32, 768])\n",
      "6752-6784\n",
      "torch.Size([32, 768])\n",
      "6784-6816\n",
      "torch.Size([32, 768])\n",
      "6816-6848\n",
      "torch.Size([32, 768])\n",
      "6848-6880\n",
      "torch.Size([32, 768])\n",
      "6880-6912\n",
      "torch.Size([32, 768])\n",
      "6912-6944\n",
      "torch.Size([32, 768])\n",
      "6944-6976\n",
      "torch.Size([32, 768])\n",
      "6976-7008\n",
      "torch.Size([32, 768])\n",
      "7008-7040\n",
      "torch.Size([32, 768])\n",
      "7040-7072\n",
      "torch.Size([32, 768])\n",
      "7072-7104\n",
      "torch.Size([32, 768])\n",
      "7104-7136\n",
      "torch.Size([32, 768])\n",
      "7136-7168\n",
      "torch.Size([32, 768])\n",
      "7168-7200\n",
      "torch.Size([32, 768])\n",
      "7200-7232\n",
      "torch.Size([32, 768])\n",
      "7232-7264\n",
      "torch.Size([32, 768])\n",
      "7264-7296\n",
      "torch.Size([32, 768])\n",
      "7296-7328\n",
      "torch.Size([32, 768])\n",
      "7328-7360\n",
      "torch.Size([32, 768])\n",
      "7360-7392\n",
      "torch.Size([32, 768])\n",
      "7392-7424\n",
      "torch.Size([32, 768])\n",
      "7424-7456\n",
      "torch.Size([32, 768])\n",
      "7456-7488\n",
      "torch.Size([32, 768])\n",
      "7488-7520\n",
      "torch.Size([32, 768])\n",
      "7520-7552\n",
      "torch.Size([32, 768])\n",
      "7552-7584\n",
      "torch.Size([32, 768])\n",
      "7584-7616\n",
      "torch.Size([32, 768])\n",
      "7616-7648\n",
      "torch.Size([32, 768])\n",
      "7648-7680\n",
      "torch.Size([32, 768])\n",
      "7680-7712\n",
      "torch.Size([32, 768])\n",
      "7712-7744\n",
      "torch.Size([32, 768])\n",
      "7744-7776\n",
      "torch.Size([32, 768])\n",
      "7776-7808\n",
      "torch.Size([32, 768])\n",
      "7808-7840\n",
      "torch.Size([32, 768])\n",
      "7840-7872\n",
      "torch.Size([32, 768])\n",
      "7872-7904\n",
      "torch.Size([32, 768])\n",
      "7904-7936\n",
      "torch.Size([32, 768])\n",
      "7936-7968\n",
      "torch.Size([32, 768])\n",
      "7968-8000\n",
      "torch.Size([32, 768])\n",
      "8000-8032\n",
      "torch.Size([32, 768])\n",
      "8032-8064\n",
      "torch.Size([32, 768])\n",
      "8064-8096\n",
      "torch.Size([32, 768])\n",
      "8096-8128\n",
      "torch.Size([32, 768])\n",
      "8128-8160\n",
      "torch.Size([32, 768])\n",
      "8160-8192\n",
      "torch.Size([32, 768])\n",
      "8192-8224\n",
      "torch.Size([32, 768])\n",
      "8224-8256\n",
      "torch.Size([32, 768])\n",
      "8256-8288\n",
      "torch.Size([32, 768])\n",
      "8288-8320\n",
      "torch.Size([32, 768])\n",
      "8320-8352\n",
      "torch.Size([32, 768])\n",
      "8352-8384\n",
      "torch.Size([32, 768])\n",
      "8384-8416\n",
      "torch.Size([32, 768])\n",
      "8416-8448\n",
      "torch.Size([32, 768])\n",
      "8448-8480\n",
      "torch.Size([32, 768])\n",
      "8480-8512\n",
      "torch.Size([32, 768])\n",
      "8512-8544\n",
      "torch.Size([32, 768])\n",
      "8544-8576\n",
      "torch.Size([32, 768])\n",
      "8576-8608\n",
      "torch.Size([32, 768])\n",
      "8608-8640\n",
      "torch.Size([32, 768])\n",
      "8640-8672\n",
      "torch.Size([32, 768])\n",
      "8672-8704\n",
      "torch.Size([32, 768])\n",
      "8704-8736\n",
      "torch.Size([32, 768])\n",
      "8736-8768\n",
      "torch.Size([32, 768])\n",
      "8768-8800\n",
      "torch.Size([32, 768])\n",
      "8800-8832\n",
      "torch.Size([32, 768])\n",
      "8832-8864\n",
      "torch.Size([32, 768])\n",
      "8864-8896\n",
      "torch.Size([32, 768])\n",
      "8896-8928\n",
      "torch.Size([32, 768])\n",
      "8928-8960\n",
      "torch.Size([32, 768])\n",
      "8960-8992\n",
      "torch.Size([32, 768])\n",
      "8992-9024\n",
      "torch.Size([32, 768])\n",
      "9024-9056\n",
      "torch.Size([32, 768])\n",
      "9056-9088\n",
      "torch.Size([32, 768])\n",
      "9088-9120\n",
      "torch.Size([32, 768])\n",
      "9120-9152\n",
      "torch.Size([32, 768])\n",
      "9152-9184\n",
      "torch.Size([32, 768])\n",
      "9184-9216\n",
      "torch.Size([32, 768])\n",
      "9216-9248\n",
      "torch.Size([32, 768])\n",
      "9248-9280\n",
      "torch.Size([32, 768])\n",
      "9280-9312\n",
      "torch.Size([32, 768])\n",
      "9312-9344\n",
      "torch.Size([32, 768])\n",
      "9344-9376\n",
      "torch.Size([32, 768])\n",
      "9376-9408\n",
      "torch.Size([32, 768])\n",
      "9408-9440\n",
      "torch.Size([32, 768])\n",
      "9440-9472\n",
      "torch.Size([32, 768])\n",
      "9472-9504\n",
      "torch.Size([32, 768])\n",
      "9504-9536\n",
      "torch.Size([32, 768])\n",
      "9536-9568\n",
      "torch.Size([32, 768])\n",
      "9568-9600\n",
      "torch.Size([32, 768])\n",
      "9600-9632\n",
      "torch.Size([32, 768])\n",
      "9632-9664\n",
      "torch.Size([32, 768])\n",
      "9664-9696\n",
      "torch.Size([32, 768])\n",
      "9696-9728\n",
      "torch.Size([32, 768])\n",
      "9728-9760\n",
      "torch.Size([32, 768])\n",
      "9760-9792\n",
      "torch.Size([32, 768])\n",
      "9792-9824\n",
      "torch.Size([32, 768])\n",
      "9824-9856\n",
      "torch.Size([32, 768])\n",
      "9856-9888\n",
      "torch.Size([32, 768])\n",
      "9888-9920\n",
      "torch.Size([32, 768])\n",
      "9920-9952\n",
      "torch.Size([32, 768])\n",
      "9952-9984\n",
      "torch.Size([32, 768])\n",
      "9984-10016\n",
      "torch.Size([32, 768])\n",
      "10016-10048\n",
      "torch.Size([32, 768])\n",
      "10048-10080\n",
      "torch.Size([32, 768])\n",
      "10080-10112\n",
      "torch.Size([32, 768])\n",
      "10112-10144\n",
      "torch.Size([32, 768])\n",
      "10144-10176\n",
      "torch.Size([32, 768])\n",
      "10176-10208\n",
      "torch.Size([32, 768])\n",
      "10208-10240\n",
      "torch.Size([32, 768])\n",
      "10240-10272\n",
      "torch.Size([32, 768])\n",
      "10272-10304\n",
      "torch.Size([32, 768])\n",
      "10304-10336\n",
      "torch.Size([32, 768])\n",
      "10336-10368\n",
      "torch.Size([32, 768])\n",
      "10368-10400\n",
      "torch.Size([32, 768])\n",
      "10400-10432\n",
      "torch.Size([32, 768])\n",
      "10432-10464\n",
      "torch.Size([32, 768])\n",
      "10464-10496\n",
      "torch.Size([32, 768])\n",
      "10496-10528\n",
      "torch.Size([32, 768])\n",
      "10528-10560\n",
      "torch.Size([32, 768])\n",
      "10560-10592\n",
      "torch.Size([32, 768])\n",
      "10592-10624\n",
      "torch.Size([32, 768])\n",
      "10624-10656\n",
      "torch.Size([32, 768])\n",
      "10656-10688\n",
      "torch.Size([32, 768])\n",
      "10688-10720\n",
      "torch.Size([32, 768])\n",
      "10720-10752\n",
      "torch.Size([32, 768])\n",
      "10752-10784\n",
      "torch.Size([32, 768])\n",
      "10784-10816\n",
      "torch.Size([32, 768])\n",
      "10816-10848\n",
      "torch.Size([32, 768])\n",
      "10848-10880\n",
      "torch.Size([32, 768])\n",
      "10880-10912\n",
      "torch.Size([32, 768])\n",
      "10912-10944\n",
      "torch.Size([32, 768])\n",
      "10944-10976\n",
      "torch.Size([32, 768])\n",
      "10976-11008\n",
      "torch.Size([32, 768])\n",
      "11008-11040\n",
      "torch.Size([32, 768])\n",
      "11040-11072\n",
      "torch.Size([32, 768])\n",
      "11072-11104\n",
      "torch.Size([32, 768])\n",
      "11104-11136\n",
      "torch.Size([32, 768])\n",
      "11136-11168\n",
      "torch.Size([32, 768])\n",
      "11168-11200\n",
      "torch.Size([32, 768])\n",
      "11200-11232\n",
      "torch.Size([32, 768])\n",
      "11232-11264\n",
      "torch.Size([32, 768])\n",
      "11264-11296\n",
      "torch.Size([32, 768])\n",
      "11296-11328\n",
      "torch.Size([32, 768])\n",
      "11328-11360\n",
      "torch.Size([32, 768])\n",
      "11360-11392\n",
      "torch.Size([32, 768])\n",
      "11392-11424\n",
      "torch.Size([32, 768])\n",
      "11424-11456\n",
      "torch.Size([32, 768])\n",
      "11456-11488\n",
      "torch.Size([32, 768])\n",
      "11488-11520\n",
      "torch.Size([32, 768])\n",
      "11520-11552\n",
      "torch.Size([32, 768])\n",
      "11552-11584\n",
      "torch.Size([32, 768])\n",
      "11584-11616\n",
      "torch.Size([32, 768])\n",
      "11616-11648\n",
      "torch.Size([32, 768])\n",
      "11648-11680\n",
      "torch.Size([32, 768])\n",
      "11680-11712\n",
      "torch.Size([32, 768])\n",
      "11712-11744\n",
      "torch.Size([32, 768])\n",
      "11744-11776\n",
      "torch.Size([32, 768])\n",
      "11776-11808\n",
      "torch.Size([32, 768])\n",
      "11808-11840\n",
      "torch.Size([32, 768])\n",
      "11840-11872\n",
      "torch.Size([32, 768])\n",
      "11872-11904\n",
      "torch.Size([32, 768])\n",
      "11904-11936\n",
      "torch.Size([32, 768])\n",
      "11936-11968\n",
      "torch.Size([32, 768])\n",
      "11968-12000\n",
      "torch.Size([32, 768])\n",
      "12000-12032\n",
      "torch.Size([32, 768])\n",
      "12032-12064\n",
      "torch.Size([32, 768])\n",
      "12064-12096\n",
      "torch.Size([32, 768])\n",
      "12096-12128\n",
      "torch.Size([5, 768])\n"
     ]
    }
   ],
   "source": [
    "if not os.path.isdir('./text_temp'):\n",
    "    os.mkdir(\"./text_temp\")\n",
    "\n",
    "batch=32\n",
    "debug_sentences=clean_sentences\n",
    "n=len(debug_sentences)//batch  #cuda OOM\n",
    "\n",
    "assert len(debug_sentences)>=batch\n",
    "\n",
    "if not os.path.isdir('./text_temp'):\n",
    "    os.mkdir(\"./text_temp\")\n",
    "\n",
    "for i in range(n if len(debug_sentences)%batch==0 else n+1):\n",
    "    print(f\"{i*batch}-{(i+1)*batch}\")\n",
    "\n",
    "    i_batch=debug_sentences[i*batch:(i+1)*batch]\n",
    "    i_ids_temp = tokenizer(i_batch, max_length=512,truncation=True,padding=\"max_length\",return_tensors='pt')\n",
    "    i_ids_temp.to(device)\n",
    "    with torch.no_grad():\n",
    "        i_bert_output = model(**i_ids_temp)\n",
    "    i_cls=i_bert_output.last_hidden_state[:,0,:].to(\"cpu\")\n",
    "    torch.save(i_cls,f\"./text_temp/{i}.pt\")\n",
    "    print(i_cls.shape)\n",
    "\n",
    "    del i_ids_temp,i_bert_output,i_cls\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "test_all=[]\n",
    "for i in range(n if len(debug_sentences)%batch==0 else n+1):\n",
    "    x=torch.load(f\"./text_temp/{i}.pt\")\n",
    "    test_all.append(x)\n",
    "\n",
    "text_features=torch.cat(test_all,dim=0)\n",
    "torch.save(text_features,os.path.join(RAW_PATH,\"text_feat.pt\"))\n",
    "\n",
    "import shutil\n",
    "shutil.rmtree(\"./text_temp\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "data": {
      "text/plain": "torch.Size([12101, 768])"
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_features.shape"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "tensor([[False, False, False,  ..., False, False, False],\n",
      "        [False, False, False,  ..., False, False, False],\n",
      "        [False, False, False,  ..., False, False, False],\n",
      "        ...,\n",
      "        [False, False, False,  ..., False, False, False],\n",
      "        [False, False, False,  ..., False, False, False],\n",
      "        [False, False, False,  ..., False, False, False]])\n"
     ]
    }
   ],
   "source": [
    "#sample random text to contrast lable\n",
    "model.to(\"cpu\")\n",
    "nnn=64\n",
    "random_i_feature=tokenizer(clean_sentences[-10:],max_length=512,truncation=True,padding=\"max_length\",return_tensors='pt')\n",
    "random_i_feature=model(**random_i_feature)\n",
    "random_i_feature=random_i_feature.last_hidden_state[:,0,:]\n",
    "\n",
    "print(torch.allclose(random_i_feature,text_features[-10:],1e-1))\n",
    "print(random_i_feature==text_features[-10:])\n",
    "\n",
    "# torch.allclose(text_features,zz,atol=1e-5)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [],
   "source": [
    "model.to(\"cpu\")\n",
    "del model\n",
    "torch.cuda.empty_cache()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Image Feature processing"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image NaN count:7\n",
      "       item_id imUrl\n",
      "1460      1460   NaN\n",
      "3578      3578   NaN\n",
      "4208      4208   NaN\n",
      "4381      4381   NaN\n",
      "4398      4398   NaN\n",
      "10771    10771   NaN\n",
      "10990    10990   NaN\n"
     ]
    }
   ],
   "source": [
    "#-------------Image Features------------------\n",
    "imUrl_na_df=i_meta_df[i_meta_df['imUrl'].isnull()]\n",
    "print(f\"image NaN count:{len(imUrl_na_df[['item_id','imUrl']])}\")\n",
    "print(imUrl_na_df[['item_id',\"imUrl\"]])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [
    {
     "data": {
      "text/plain": "0"
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_n=0\n",
    "string_n=0\n",
    "for i,row in i_meta_df.iterrows():\n",
    "    if isinstance(row['imUrl'],list):\n",
    "        list_n+=1\n",
    "    if isinstance(row['imUrl'],str):\n",
    "        string_n+=1\n",
    "list_n\n",
    "#Some origin item has many image url like:[....jpg,....jpg,....jpg]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [
    {
     "data": {
      "text/plain": "12101"
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(i_meta_df['imUrl'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [
    {
     "data": {
      "text/plain": "0        http://ecx.images-amazon.com/images/I/41Rn18Oe...\n1        http://ecx.images-amazon.com/images/I/41QWW9v1...\n2        http://ecx.images-amazon.com/images/I/51iT2k6L...\n3        http://ecx.images-amazon.com/images/I/418LYGLE...\n4        http://ecx.images-amazon.com/images/I/31L2n60J...\n                               ...                        \n12096    http://ecx.images-amazon.com/images/I/61enVb2X...\n12097    http://ecx.images-amazon.com/images/I/41q7jpgt...\n12098    http://ecx.images-amazon.com/images/I/412qdoPc...\n12099    http://ecx.images-amazon.com/images/I/31JTTyCU...\n12100    http://ecx.images-amazon.com/images/I/41up5%2B...\nName: imUrl, Length: 12101, dtype: object"
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_meta_df['imUrl']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18 http://ecx.images-amazon.com/images/I/11ZEXJHC1VL.jpg\n",
      "21 http://ecx.images-amazon.com/images/I/11F85AXJ9JL.jpg\n",
      "30 http://ecx.images-amazon.com/images/I/21HN33EM9CL.jpg\n",
      "34 http://ecx.images-amazon.com/images/I/41cO9Od7ZlL.jpg\n",
      "35 http://ecx.images-amazon.com/images/I/21S55630NJL.jpg\n",
      "36 http://ecx.images-amazon.com/images/I/212D22J613L.jpg\n",
      "41 http://ecx.images-amazon.com/images/I/1169PW0QSKL.jpg\n",
      "48 http://ecx.images-amazon.com/images/I/21FG5TC6FAL.jpg\n",
      "49 http://ecx.images-amazon.com/images/I/118CZV58ZFL.jpg\n",
      "54 http://ecx.images-amazon.com/images/I/21C6Y9AMWPL.jpg\n",
      "59 http://ecx.images-amazon.com/images/I/11TFMG6Y9NL.jpg\n",
      "60 http://ecx.images-amazon.com/images/I/21J2jTmvnsL.jpg\n",
      "68 http://ecx.images-amazon.com/images/I/11AK1M6GSPL.jpg\n",
      "87 http://ecx.images-amazon.com/images/I/11A4FZ4032L.jpg\n",
      "109 http://ecx.images-amazon.com/images/I/21MVF6ESEEL.jpg\n",
      "115 http://ecx.images-amazon.com/images/I/21DPnGiPj7L.jpg\n",
      "116 http://ecx.images-amazon.com/images/I/31xUb0FGiZL.jpg\n",
      "143 http://ecx.images-amazon.com/images/I/31ps80wLtsL._SY445_.jpg\n",
      "152 http://ecx.images-amazon.com/images/I/3104YQ0Z6KL.jpg\n",
      "155 http://ecx.images-amazon.com/images/I/11AXPR25H1L.jpg\n",
      "171 http://ecx.images-amazon.com/images/I/212W797ZXZL.jpg\n",
      "173 http://ecx.images-amazon.com/images/I/1186EAQ9K3L.jpg\n",
      "174 http://ecx.images-amazon.com/images/I/21QR7053JSL.jpg\n",
      "181 http://ecx.images-amazon.com/images/I/115DNV2RAEL.jpg\n",
      "182 http://ecx.images-amazon.com/images/I/112YQX77Q9L.jpg\n",
      "193 http://ecx.images-amazon.com/images/I/1198EEBF4NL.jpg\n",
      "203 http://ecx.images-amazon.com/images/I/216KDT5P7CL.jpg\n",
      "209 http://ecx.images-amazon.com/images/I/21RC3E4FG1L.jpg\n",
      "216 http://ecx.images-amazon.com/images/I/11VFRFDC4TL.jpg\n",
      "218 http://ecx.images-amazon.com/images/I/21cQV81s8ZL.jpg\n",
      "221 http://ecx.images-amazon.com/images/I/31N6%2BqfIkZL.jpg\n",
      "225 http://ecx.images-amazon.com/images/I/21Y99jVVogL.jpg\n",
      "226 http://ecx.images-amazon.com/images/I/21qT7aPV39L.jpg\n",
      "228 http://ecx.images-amazon.com/images/I/1104C18YZmL.jpg\n",
      "233 http://ecx.images-amazon.com/images/I/21XM4Q26YGL.jpg\n",
      "241 http://ecx.images-amazon.com/images/I/319M3qpD20L.jpg\n",
      "245 http://ecx.images-amazon.com/images/I/11CoCdd5INL.jpg\n",
      "253 http://ecx.images-amazon.com/images/I/11v3l9yMdmL.jpg\n",
      "272 http://ecx.images-amazon.com/images/I/21AYOfMzdOL.jpg\n",
      "302 http://ecx.images-amazon.com/images/I/410qg4QApeL._SX342_.jpg\n",
      "348 http://ecx.images-amazon.com/images/I/31RUkKzDKrL._SX342_.jpg\n",
      "368 http://ecx.images-amazon.com/images/I/41wpfzvot-L.jpg\n",
      "436 http://ecx.images-amazon.com/images/I/216FACY07ML.jpg\n",
      "447 http://ecx.images-amazon.com/images/I/2150GY9NZXL.jpg\n",
      "477 http://ecx.images-amazon.com/images/I/11ff8RKKWUL.jpg\n",
      "503 http://ecx.images-amazon.com/images/I/219ZX3JJP1L.jpg\n",
      "506 http://ecx.images-amazon.com/images/I/31XFMcQF1sL.jpg\n",
      "512 http://ecx.images-amazon.com/images/I/31m469dFxUL._SX342_.jpg\n",
      "514 http://ecx.images-amazon.com/images/I/31V71Axn8KL.jpg\n",
      "520 http://ecx.images-amazon.com/images/I/31pZ%2BhRvr3L.jpg\n",
      "522 http://ecx.images-amazon.com/images/I/31PCTRetMdL.jpg\n",
      "525 http://ecx.images-amazon.com/images/I/21JfTrjiuUL.jpg\n",
      "549 http://ecx.images-amazon.com/images/I/21FfpAT4m3L.jpg\n",
      "568 http://ecx.images-amazon.com/images/I/31Ktjf-FtTL.jpg\n",
      "595 http://ecx.images-amazon.com/images/I/21C4iSEPITL.jpg\n",
      "599 http://ecx.images-amazon.com/images/I/31gDubsc4LL.jpg\n",
      "600 http://ecx.images-amazon.com/images/I/314sQ97oL4L._SX342_.jpg\n",
      "617 http://ecx.images-amazon.com/images/I/21PSZvp98AL.jpg\n",
      "621 http://ecx.images-amazon.com/images/I/21cyeyYA47L.jpg\n",
      "633 http://ecx.images-amazon.com/images/I/31Z%2BXmNxilL.jpg\n",
      "634 http://ecx.images-amazon.com/images/I/41wIPX8H7ZL.jpg\n",
      "638 http://ecx.images-amazon.com/images/I/31kZAk14LpL.jpg\n",
      "683 http://ecx.images-amazon.com/images/I/21WMWK33PSL.jpg\n",
      "715 http://ecx.images-amazon.com/images/I/21WB2C27BVL.jpg\n",
      "724 http://ecx.images-amazon.com/images/I/21M7M71WWFL.jpg\n",
      "734 http://ecx.images-amazon.com/images/I/21MHCDKXYCL.jpg\n",
      "754 http://ecx.images-amazon.com/images/I/21LknPX8cXL.jpg\n",
      "767 http://ecx.images-amazon.com/images/I/216qSPj4N3L.jpg\n",
      "772 http://ecx.images-amazon.com/images/I/11AQQWA2B4L.jpg\n",
      "832 http://ecx.images-amazon.com/images/I/31TZ6xgBM-L.jpg\n",
      "834 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "880 http://ecx.images-amazon.com/images/I/41WZQwSb8aL._SX342_.jpg\n",
      "897 http://ecx.images-amazon.com/images/I/21GHBQSMWFL.jpg\n",
      "927 http://ecx.images-amazon.com/images/I/31yL8ONDt1L.jpg\n",
      "952 http://ecx.images-amazon.com/images/I/21621YRF9ZL.jpg\n",
      "961 http://ecx.images-amazon.com/images/I/21W6S7GXVTL.jpg\n",
      "977 http://ecx.images-amazon.com/images/I/11pLdM2nPTL.jpg\n",
      "979 http://ecx.images-amazon.com/images/I/21V1vLF1xvL.jpg\n",
      "982 http://ecx.images-amazon.com/images/I/21kLgg7%2BTaL.jpg\n",
      "986 http://ecx.images-amazon.com/images/I/41SQD8VM86L.jpg\n",
      "994 http://ecx.images-amazon.com/images/I/211XWocl2vL.jpg\n",
      "995 http://ecx.images-amazon.com/images/I/31L44D70UNL.jpg\n",
      "1001 http://ecx.images-amazon.com/images/I/21EJX%2BZOsaL.jpg\n",
      "1123 http://ecx.images-amazon.com/images/I/21quQosWAKL.jpg\n",
      "1200 http://ecx.images-amazon.com/images/I/11CPT53DQ0L.jpg\n",
      "1223 http://ecx.images-amazon.com/images/I/31h%2Bc-4vVmL.jpg\n",
      "1224 http://ecx.images-amazon.com/images/I/31vKyrV5RnL.jpg\n",
      "1249 http://ecx.images-amazon.com/images/I/21DC83F7NHL.jpg\n",
      "1267 http://ecx.images-amazon.com/images/I/215s1YAnTBL.jpg\n",
      "1274 http://ecx.images-amazon.com/images/I/31zkwjDe3AL._SX342_.jpg\n",
      "1283 http://ecx.images-amazon.com/images/I/21hXCw%2BeBYL.jpg\n",
      "1309 http://ecx.images-amazon.com/images/I/21MVF6ESEEL.jpg\n",
      "1343 http://ecx.images-amazon.com/images/I/21DNC87qBPL.jpg\n",
      "1346 http://ecx.images-amazon.com/images/I/41EARXKK40L.gif\n",
      "1355 http://ecx.images-amazon.com/images/I/211H9nr1JdL.jpg\n",
      "1516 http://ecx.images-amazon.com/images/I/31UJaKBHzhL.jpg\n",
      "1534 http://ecx.images-amazon.com/images/I/21uJGsAAXEL.jpg\n",
      "1562 http://ecx.images-amazon.com/images/I/11t8buUUrBL.jpg\n",
      "1564 http://ecx.images-amazon.com/images/I/316LKRL6zeL.jpg\n",
      "1611 http://ecx.images-amazon.com/images/I/2179gD3BrBL.jpg\n",
      "1613 http://ecx.images-amazon.com/images/I/41HBxPjd3-L.jpg\n",
      "1654 http://ecx.images-amazon.com/images/I/31Rk4ftwkBL.jpg\n",
      "1671 http://ecx.images-amazon.com/images/I/31A01RYiSaL.jpg\n",
      "1672 http://ecx.images-amazon.com/images/I/319DHrMVSjL.jpg\n",
      "1676 http://ecx.images-amazon.com/images/I/41drqOYX77L.jpg\n",
      "1697 http://ecx.images-amazon.com/images/I/41b4Gm0BQyL._SX342_.jpg\n",
      "1720 http://ecx.images-amazon.com/images/I/21vnRO0xAoL.jpg\n",
      "1736 http://ecx.images-amazon.com/images/I/31otfl35W-L.jpg\n",
      "1740 http://ecx.images-amazon.com/images/I/21VA2ZoMANL.jpg\n",
      "1788 http://ecx.images-amazon.com/images/I/21YCVaIPAEL.jpg\n",
      "1794 http://ecx.images-amazon.com/images/I/310E8i8Y3uL.jpg\n",
      "1797 http://ecx.images-amazon.com/images/I/11ERlpVc59L.jpg\n",
      "1803 http://ecx.images-amazon.com/images/I/31kNx1aBr5L.jpg\n",
      "1808 http://ecx.images-amazon.com/images/I/11AyBAyZ7cL.jpg\n",
      "1810 http://ecx.images-amazon.com/images/I/21wdkhbcmPL.jpg\n",
      "1817 http://ecx.images-amazon.com/images/I/21lxyfl1EXL.jpg\n",
      "1820 http://ecx.images-amazon.com/images/I/31Opeqru-%2BL.jpg\n",
      "1834 http://ecx.images-amazon.com/images/I/21VK3cB%2BoHL.jpg\n",
      "1850 http://ecx.images-amazon.com/images/I/31foZYppB9L.jpg\n",
      "1862 http://ecx.images-amazon.com/images/I/31pN-Lolu2L.jpg\n",
      "1867 http://ecx.images-amazon.com/images/I/21H8T%2BM4QLL.jpg\n",
      "1869 http://ecx.images-amazon.com/images/I/21-sUyrPwYL.jpg\n",
      "1873 http://ecx.images-amazon.com/images/I/61pNsVQxpjL.gif\n",
      "1876 http://ecx.images-amazon.com/images/I/11BCGeodD4L.jpg\n",
      "1893 http://ecx.images-amazon.com/images/I/31y8TEkZpIL.jpg\n",
      "1894 http://ecx.images-amazon.com/images/I/21FARoIkWPL.jpg\n",
      "1897 http://ecx.images-amazon.com/images/I/21Hya1IQOlL.jpg\n",
      "1918 http://ecx.images-amazon.com/images/I/216a4uDUaHL.jpg\n",
      "1919 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "1980 http://ecx.images-amazon.com/images/I/41Yp3vuDtnL.jpg\n",
      "1981 http://ecx.images-amazon.com/images/I/41ziZ30o4SL.jpg\n",
      "1990 http://ecx.images-amazon.com/images/I/31McQEcV8mL.jpg\n",
      "2008 http://ecx.images-amazon.com/images/I/21W1FXP99EL.jpg\n",
      "2014 http://ecx.images-amazon.com/images/I/21GNM1h9HPL.jpg\n",
      "2027 http://ecx.images-amazon.com/images/I/312CUZvw6DL.jpg\n",
      "2030 http://ecx.images-amazon.com/images/I/21ZNAgEihdL.jpg\n",
      "2038 http://ecx.images-amazon.com/images/I/31hNHeNHyEL.jpg\n",
      "2039 http://ecx.images-amazon.com/images/I/31-4zfZ62uL.jpg\n",
      "2048 http://ecx.images-amazon.com/images/I/11gjBpgbIfL.jpg\n",
      "2050 http://ecx.images-amazon.com/images/I/313cXO6u3ZL.jpg\n",
      "2072 http://ecx.images-amazon.com/images/I/41peu7TL9XL._SX395_.jpg\n",
      "2078 http://ecx.images-amazon.com/images/I/41A1n7Nm5wL.jpg\n",
      "2079 http://ecx.images-amazon.com/images/I/41ScXHbxrkL.jpg\n",
      "2090 http://ecx.images-amazon.com/images/I/21xw-dldKFL.jpg\n",
      "2095 http://ecx.images-amazon.com/images/I/21Qap1chfbL.jpg\n",
      "2097 http://ecx.images-amazon.com/images/I/31HJGbVBmLL.jpg\n",
      "2099 http://ecx.images-amazon.com/images/I/11Sql0OQwqL.jpg\n",
      "2119 http://ecx.images-amazon.com/images/I/21H7Z1JYAbL.jpg\n",
      "2120 http://ecx.images-amazon.com/images/I/21b33uGHM-L.jpg\n",
      "2166 http://ecx.images-amazon.com/images/I/51Er6IJ2pSL._SY445_.jpg\n",
      "2187 http://ecx.images-amazon.com/images/I/11eIJ0TRmaL.jpg\n",
      "2192 http://ecx.images-amazon.com/images/I/31heNI-PR5L.jpg\n",
      "2244 http://ecx.images-amazon.com/images/I/31PMVVbm2mL.jpg\n",
      "2251 http://ecx.images-amazon.com/images/I/412CSL0gK2L.jpg\n",
      "2252 http://ecx.images-amazon.com/images/I/21mJD%2B%2BmAGL.jpg\n",
      "2257 http://ecx.images-amazon.com/images/I/31pBBspLMBL.jpg\n",
      "2281 http://ecx.images-amazon.com/images/I/21WTEJ43A-L.jpg\n",
      "2324 http://ecx.images-amazon.com/images/I/21Vj6FzkHUL.jpg\n",
      "2326 http://ecx.images-amazon.com/images/I/118V5SjhsiL.jpg\n",
      "2333 http://ecx.images-amazon.com/images/I/210B9UaH9NL.jpg\n",
      "2336 http://ecx.images-amazon.com/images/I/31YU1KRKh8L.jpg\n",
      "2360 http://ecx.images-amazon.com/images/I/41A16EHceOL.jpg\n",
      "2375 http://ecx.images-amazon.com/images/I/21z6oi5SflL.jpg\n",
      "2376 http://ecx.images-amazon.com/images/I/11mW3A2JX5L.jpg\n",
      "2426 http://ecx.images-amazon.com/images/I/31znIdECPdL.jpg\n",
      "2438 http://ecx.images-amazon.com/images/I/218M2LWE5xL.jpg\n",
      "2441 http://ecx.images-amazon.com/images/I/31OFlYvULpL.jpg\n",
      "2443 http://ecx.images-amazon.com/images/I/21tAuOfLPgL.jpg\n",
      "2448 http://ecx.images-amazon.com/images/I/31OHzJbK2GL.jpg\n",
      "2449 http://ecx.images-amazon.com/images/I/21l5GVAu61L.jpg\n",
      "2466 http://ecx.images-amazon.com/images/I/31QlT6udYHL.jpg\n",
      "2468 http://ecx.images-amazon.com/images/I/21ti8bDrO2L.jpg\n",
      "2477 http://ecx.images-amazon.com/images/I/31eND6uRiVL.jpg\n",
      "2524 http://ecx.images-amazon.com/images/I/31hmgyK2t-L.jpg\n",
      "2538 http://ecx.images-amazon.com/images/I/41Mwqa%2BLvoL.jpg\n",
      "2542 http://ecx.images-amazon.com/images/I/21ljTVJSOfL.jpg\n",
      "2556 http://ecx.images-amazon.com/images/I/216vSrqFc2L.jpg\n",
      "2570 http://ecx.images-amazon.com/images/I/21lXb-LR-vL.jpg\n",
      "2599 http://ecx.images-amazon.com/images/I/21OGKV8DnpL.jpg\n",
      "2614 http://ecx.images-amazon.com/images/I/31qLJpwfUSL.jpg\n",
      "2641 http://ecx.images-amazon.com/images/I/211FhXUxIBL.jpg\n",
      "2676 http://ecx.images-amazon.com/images/I/31kstpai-GL.jpg\n",
      "2694 http://ecx.images-amazon.com/images/I/313jBBq7KfL.jpg\n",
      "2702 http://ecx.images-amazon.com/images/I/11XgAllFNQL.jpg\n",
      "2733 http://ecx.images-amazon.com/images/I/21sxLq2pGCL.jpg\n",
      "2751 http://ecx.images-amazon.com/images/I/214knQIWKlL.jpg\n",
      "2755 http://ecx.images-amazon.com/images/I/31uOEJkM4ML.jpg\n",
      "2786 http://ecx.images-amazon.com/images/I/21w5zHnc-aL.jpg\n",
      "2791 http://ecx.images-amazon.com/images/I/31TVX62XSeL.jpg\n",
      "2817 http://ecx.images-amazon.com/images/I/31gBFxyk2DL.jpg\n",
      "2880 http://ecx.images-amazon.com/images/I/41VyPBnL2OL.jpg\n",
      "2968 http://ecx.images-amazon.com/images/I/31TZ4CDN01L.jpg\n",
      "2987 http://ecx.images-amazon.com/images/I/51UmJ85cLWL.gif\n",
      "3008 http://ecx.images-amazon.com/images/I/21nuwlCuELL.jpg\n",
      "3010 http://ecx.images-amazon.com/images/I/31aXseQ9ggL.jpg\n",
      "3030 http://ecx.images-amazon.com/images/I/31IMHeZtcFL.jpg\n",
      "3032 http://ecx.images-amazon.com/images/I/41jRb%2B81gIL.jpg\n",
      "3033 http://ecx.images-amazon.com/images/I/31-xW8gZdyL.jpg\n",
      "3082 http://ecx.images-amazon.com/images/I/21ZpqvU%2BnAL.jpg\n",
      "3125 http://ecx.images-amazon.com/images/I/31RACl50kmL.jpg\n",
      "3134 http://ecx.images-amazon.com/images/I/31CgKl4vAOL.jpg\n",
      "3186 http://ecx.images-amazon.com/images/I/11MPVjMaHXL.jpg\n",
      "3189 http://ecx.images-amazon.com/images/I/31wNBhF8QQL._SY445_.jpg\n",
      "3207 http://ecx.images-amazon.com/images/I/31A0Vka3CeL.jpg\n",
      "3210 http://ecx.images-amazon.com/images/I/11vkZyVvz1L.jpg\n",
      "3213 http://ecx.images-amazon.com/images/I/41666gYrrsL.jpg\n",
      "3231 http://ecx.images-amazon.com/images/I/11p88nOuoyL.jpg\n",
      "3232 http://ecx.images-amazon.com/images/I/314htyuF2PL.jpg\n",
      "3288 http://ecx.images-amazon.com/images/I/21-zHyXSKQL.jpg\n",
      "3302 http://ecx.images-amazon.com/images/I/31XJzD0AtPL.jpg\n",
      "3306 http://ecx.images-amazon.com/images/I/31AGgPgg2yL.jpg\n",
      "3323 http://ecx.images-amazon.com/images/I/21Ecu6dQUIL.jpg\n",
      "3332 http://ecx.images-amazon.com/images/I/313JlYZCUgL.jpg\n",
      "3338 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "3339 http://ecx.images-amazon.com/images/I/31FJ7yZg6mL._SX342_.jpg\n",
      "3341 http://ecx.images-amazon.com/images/I/41Gloz2Et4L._SX342_.jpg\n",
      "3361 http://ecx.images-amazon.com/images/I/41%2BfDI-1JZL.jpg\n",
      "3396 http://ecx.images-amazon.com/images/I/31R8L0d3ZzL.jpg\n",
      "3421 http://ecx.images-amazon.com/images/I/21JiRiRRFqL.jpg\n",
      "3423 http://ecx.images-amazon.com/images/I/41JgmmzjzyL.jpg\n",
      "3424 http://ecx.images-amazon.com/images/I/21FpLBP%2BlkL.jpg\n",
      "3428 http://ecx.images-amazon.com/images/I/31MjK5gqN1L.jpg\n",
      "3430 http://ecx.images-amazon.com/images/I/214yIvf9lLL.jpg\n",
      "3448 http://ecx.images-amazon.com/images/I/21DIOvfUWAL.jpg\n",
      "3459 http://ecx.images-amazon.com/images/I/21f5rKPJjcL.jpg\n",
      "3460 http://ecx.images-amazon.com/images/I/2136qCqm-YL.jpg\n",
      "3481 http://ecx.images-amazon.com/images/I/21iepe0SllL.jpg\n",
      "3488 http://ecx.images-amazon.com/images/I/21wl5B28P5L.jpg\n",
      "3490 http://ecx.images-amazon.com/images/I/316UeumXSqL.jpg\n",
      "3491 http://ecx.images-amazon.com/images/I/413vDcB6blL.jpg\n",
      "3525 http://ecx.images-amazon.com/images/I/31xHDjK9eVL.jpg\n",
      "3544 http://ecx.images-amazon.com/images/I/113W02Kc7eL.jpg\n",
      "3548 http://ecx.images-amazon.com/images/I/318kGIGsXSL.jpg\n",
      "3549 http://ecx.images-amazon.com/images/I/21veI9xodNL.jpg\n",
      "3552 http://ecx.images-amazon.com/images/I/61lmxuCweaL._SX342_.jpg\n",
      "3553 http://ecx.images-amazon.com/images/I/51gLyIRdnYL._SY445_.jpg\n",
      "3562 http://ecx.images-amazon.com/images/I/31BD8yHCDzL.jpg\n",
      "3567 http://ecx.images-amazon.com/images/I/21nqQipWA6L.jpg\n",
      "3568 http://ecx.images-amazon.com/images/I/31zzzcZXArL.jpg\n",
      "3595 http://ecx.images-amazon.com/images/I/116gR8SANjL.jpg\n",
      "3620 http://ecx.images-amazon.com/images/I/21U3Waam12L.jpg\n",
      "3632 http://ecx.images-amazon.com/images/I/21Fp1vxchFL.jpg\n",
      "3668 http://ecx.images-amazon.com/images/I/21MVF6ESEEL.jpg\n",
      "3673 http://ecx.images-amazon.com/images/I/31CNCVETGSL.jpg\n",
      "3675 http://ecx.images-amazon.com/images/I/21S55630NJL.jpg\n",
      "3676 http://ecx.images-amazon.com/images/I/11VBHMD42EL.jpg\n",
      "3682 http://ecx.images-amazon.com/images/I/21Q55NQMNZL.jpg\n",
      "3686 http://ecx.images-amazon.com/images/I/11WK2ZQYDCL.jpg\n",
      "3689 http://ecx.images-amazon.com/images/I/21WK08XYNHL.jpg\n",
      "3691 http://ecx.images-amazon.com/images/I/11D57TXQMVL.jpg\n",
      "3734 http://ecx.images-amazon.com/images/I/31NzzKIq7lL.jpg\n",
      "3736 http://ecx.images-amazon.com/images/I/21qSBTSZyQL.jpg\n",
      "3754 http://ecx.images-amazon.com/images/I/21a4HZHgWuL.jpg\n",
      "3764 http://ecx.images-amazon.com/images/I/11EHCMXNT0L.jpg\n",
      "3778 http://ecx.images-amazon.com/images/I/21f89c8XHbL.jpg\n",
      "3829 http://ecx.images-amazon.com/images/I/11dbIjjfeOL.jpg\n",
      "3857 http://ecx.images-amazon.com/images/I/21TC1Z4JC6L.jpg\n",
      "3866 http://ecx.images-amazon.com/images/I/213hDR-xs4L.jpg\n",
      "3874 http://ecx.images-amazon.com/images/I/21YwenxQdUL.jpg\n",
      "3890 http://ecx.images-amazon.com/images/I/31yKs1zox9L.jpg\n",
      "3891 http://ecx.images-amazon.com/images/I/21Wi-crVBeL.jpg\n",
      "3896 http://ecx.images-amazon.com/images/I/21vTa-PzqIL.jpg\n",
      "3918 http://ecx.images-amazon.com/images/I/11G1EAGW13L.jpg\n",
      "3944 http://ecx.images-amazon.com/images/I/31TMqzTVJYL.jpg\n",
      "3970 http://ecx.images-amazon.com/images/I/2182VwZrJtL.jpg\n",
      "3972 http://ecx.images-amazon.com/images/I/31ufeOtg9RL.jpg\n",
      "3976 http://ecx.images-amazon.com/images/I/21oUWvHonXL.jpg\n",
      "3989 http://ecx.images-amazon.com/images/I/31R4IcI9iPL.jpg\n",
      "3990 http://ecx.images-amazon.com/images/I/41e9kIGqEBL._SX395_.jpg\n",
      "4060 http://ecx.images-amazon.com/images/I/21U0rQoDC5L.jpg\n",
      "4064 http://ecx.images-amazon.com/images/I/21NflgXHM0L.jpg\n",
      "4076 http://ecx.images-amazon.com/images/I/31YN8HkuDCL.jpg\n",
      "4077 http://ecx.images-amazon.com/images/I/01ugGgWA7nL.jpg\n",
      "4078 http://ecx.images-amazon.com/images/I/31aw%2BglSDfL.jpg\n",
      "4080 http://ecx.images-amazon.com/images/I/41%2BNtp73K6L.jpg\n",
      "4085 http://ecx.images-amazon.com/images/I/11tFteEG-DL.jpg\n",
      "4106 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "4136 http://ecx.images-amazon.com/images/I/41LtCudyWDL.jpg\n",
      "4193 http://ecx.images-amazon.com/images/I/31ThRnB2xQL.jpg\n",
      "4194 http://ecx.images-amazon.com/images/I/31904O9-KjL.jpg\n",
      "4206 http://ecx.images-amazon.com/images/I/31yajl1bfOL.jpg\n",
      "4207 http://ecx.images-amazon.com/images/I/11BblHnnK5L.jpg\n",
      "4223 http://ecx.images-amazon.com/images/I/31RT9rhFqAL.jpg\n",
      "4264 http://ecx.images-amazon.com/images/I/31eWE0tFZuL.jpg\n",
      "4276 http://ecx.images-amazon.com/images/I/31j3Ou0c4iL.jpg\n",
      "4308 http://ecx.images-amazon.com/images/I/21Pidck7ONL.jpg\n",
      "4345 http://ecx.images-amazon.com/images/I/31i-kh0Cp9L.jpg\n",
      "4378 http://ecx.images-amazon.com/images/I/11Gd%2BtMNSbL.jpg\n",
      "4382 http://ecx.images-amazon.com/images/I/31tFN3GULZL.jpg\n",
      "4389 http://ecx.images-amazon.com/images/I/41Fq1pJIc8L.jpg\n",
      "4413 http://ecx.images-amazon.com/images/I/215oXD0XwIL.jpg\n",
      "4415 http://ecx.images-amazon.com/images/I/11jjJBN98bL.jpg\n",
      "4417 http://ecx.images-amazon.com/images/I/11eF6S8ps9L.jpg\n",
      "4418 http://ecx.images-amazon.com/images/I/21Yl49O7xRL.jpg\n",
      "4447 http://ecx.images-amazon.com/images/I/41dBDAwQTjL.jpg\n",
      "4450 http://ecx.images-amazon.com/images/I/31ZjuCbx7ML.jpg\n",
      "4476 http://ecx.images-amazon.com/images/I/21roJc0TkuL.jpg\n",
      "4513 http://ecx.images-amazon.com/images/I/21LBIO%2BYvuL.jpg\n",
      "4546 http://ecx.images-amazon.com/images/I/21gGVmUE3AL.jpg\n",
      "4548 http://ecx.images-amazon.com/images/I/21aQAj6IX3L.jpg\n",
      "4585 http://ecx.images-amazon.com/images/I/3101K3KPOxL.jpg\n",
      "4595 http://ecx.images-amazon.com/images/I/21s84LbwJSL.jpg\n",
      "4634 http://ecx.images-amazon.com/images/I/116v-OHQw1L.jpg\n",
      "4635 http://ecx.images-amazon.com/images/I/31PXn69mJBL.jpg\n",
      "4648 http://ecx.images-amazon.com/images/I/31E46wBk%2BaL.jpg\n",
      "4688 http://ecx.images-amazon.com/images/I/31MEsRocADL.jpg\n",
      "4689 http://ecx.images-amazon.com/images/I/31yDEpLNpkL.jpg\n",
      "4703 http://ecx.images-amazon.com/images/I/21YmwXm-hJL.jpg\n",
      "4710 http://ecx.images-amazon.com/images/I/31YtPOJ06UL.jpg\n",
      "4735 http://ecx.images-amazon.com/images/I/51HAAC3J0jL._SX342_.jpg\n",
      "4737 http://ecx.images-amazon.com/images/I/41pqcoouGxL._SX342_.jpg\n",
      "4746 http://ecx.images-amazon.com/images/I/41qFM1Dw-wL._SX342_.jpg\n",
      "4760 http://ecx.images-amazon.com/images/I/21Z57K35P3L.jpg\n",
      "4788 http://ecx.images-amazon.com/images/I/31rj7ph%2BJML.jpg\n",
      "4790 http://ecx.images-amazon.com/images/I/11lrfbddAqL.jpg\n",
      "4791 http://ecx.images-amazon.com/images/I/11p5xkxMN0L.jpg\n",
      "4795 http://ecx.images-amazon.com/images/I/11CUPp9B0NL.jpg\n",
      "4796 http://ecx.images-amazon.com/images/I/11wMBWsO44L.jpg\n",
      "4815 http://ecx.images-amazon.com/images/I/413%2B6miPwDL._SX342_.jpg\n",
      "4837 http://ecx.images-amazon.com/images/I/21MIX%2BABVIL.jpg\n",
      "4873 http://ecx.images-amazon.com/images/I/21yO-Vk-scL.jpg\n",
      "4879 http://ecx.images-amazon.com/images/I/31zAO-1ut-L.jpg\n",
      "4912 http://ecx.images-amazon.com/images/I/21ayd9AvUIL.jpg\n",
      "4925 http://ecx.images-amazon.com/images/I/41PrtF6nfbL.jpg\n",
      "4933 http://ecx.images-amazon.com/images/I/11PqhBDBDLL.jpg\n",
      "4949 http://ecx.images-amazon.com/images/I/31jStx95gFL.jpg\n",
      "4965 http://ecx.images-amazon.com/images/I/31x0yLHZ-jL.jpg\n",
      "4981 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "4984 http://ecx.images-amazon.com/images/I/21MmJmhhGtL.jpg\n",
      "5011 http://ecx.images-amazon.com/images/I/21zGmo3OtqL.jpg\n",
      "5012 http://ecx.images-amazon.com/images/I/31KT9R8dbnL.jpg\n",
      "5013 http://ecx.images-amazon.com/images/I/21RaKOv5ABL.jpg\n",
      "5041 http://ecx.images-amazon.com/images/I/219c-M-qyfL.jpg\n",
      "5057 http://ecx.images-amazon.com/images/I/31l4YfB8qTL.jpg\n",
      "5101 http://ecx.images-amazon.com/images/I/21WI6HkcE%2BL.jpg\n",
      "5122 http://ecx.images-amazon.com/images/I/31IUOjPnZdL.jpg\n",
      "5125 http://ecx.images-amazon.com/images/I/317IKYVXtSL.jpg\n",
      "5138 http://ecx.images-amazon.com/images/I/21ZFoAlx6sL.jpg\n",
      "5162 http://ecx.images-amazon.com/images/I/41j-CZSBrnL.jpg\n",
      "5182 http://ecx.images-amazon.com/images/I/11GCUFJBzGL.jpg\n",
      "5202 http://ecx.images-amazon.com/images/I/31wuYURwwEL.jpg\n",
      "5276 http://ecx.images-amazon.com/images/I/11hJr9BqwDL.jpg\n",
      "5278 http://ecx.images-amazon.com/images/I/214VyLg%2BdiL.jpg\n",
      "5285 http://ecx.images-amazon.com/images/I/31wMwiHwfIL.jpg\n",
      "5300 http://ecx.images-amazon.com/images/I/31dYnRjnjEL.jpg\n",
      "5350 http://ecx.images-amazon.com/images/I/21fIU1Tt94L.jpg\n",
      "5351 http://ecx.images-amazon.com/images/I/21a-cwz22jL.jpg\n",
      "5363 http://ecx.images-amazon.com/images/I/21qPRo36NxL.jpg\n",
      "5381 http://ecx.images-amazon.com/images/I/416kBPMLd2L.jpg\n",
      "5408 http://ecx.images-amazon.com/images/I/31v1Y%2BhLTGL._SX342_.jpg\n",
      "5420 http://ecx.images-amazon.com/images/I/41iLcfa3bYL._SX342_.jpg\n",
      "5493 http://ecx.images-amazon.com/images/I/31mYnMc1iML.jpg\n",
      "5513 http://ecx.images-amazon.com/images/I/21JdHhH2pDL.jpg\n",
      "5515 http://ecx.images-amazon.com/images/I/2139cVe-aJL.jpg\n",
      "5524 http://ecx.images-amazon.com/images/I/216zkIIoQlL.jpg\n",
      "5531 http://ecx.images-amazon.com/images/I/11VdDpaTYIL.jpg\n",
      "5534 http://ecx.images-amazon.com/images/I/41eRKZuTR2L.jpg\n",
      "5542 http://ecx.images-amazon.com/images/I/21kljzXHm3L.jpg\n",
      "5592 http://ecx.images-amazon.com/images/I/21q3LGx6bLL.jpg\n",
      "5613 http://ecx.images-amazon.com/images/I/31UmxGrCiQL.jpg\n",
      "5619 http://ecx.images-amazon.com/images/I/21HXh4Gs2rL.jpg\n",
      "5620 http://ecx.images-amazon.com/images/I/31ivmYmlAuL.jpg\n",
      "5653 http://ecx.images-amazon.com/images/I/31rHffKZw%2BL.jpg\n",
      "5664 http://ecx.images-amazon.com/images/I/21Kju3RI4TL.jpg\n",
      "5665 http://ecx.images-amazon.com/images/I/11HngoS2sPL.jpg\n",
      "5667 http://ecx.images-amazon.com/images/I/31AGBBVZf3L.jpg\n",
      "5672 http://ecx.images-amazon.com/images/I/115N6tDcSvL.jpg\n",
      "5694 http://ecx.images-amazon.com/images/I/212emYyIsrL.jpg\n",
      "5695 http://ecx.images-amazon.com/images/I/11hoQwI1QKL.jpg\n",
      "5697 http://ecx.images-amazon.com/images/I/21sbh-1F1GL.jpg\n",
      "5704 http://ecx.images-amazon.com/images/I/21RhNZrZsAL.jpg\n",
      "5719 http://ecx.images-amazon.com/images/I/21VtGZasxvL.jpg\n",
      "5723 http://ecx.images-amazon.com/images/I/218jxceYlmL.jpg\n",
      "5772 http://ecx.images-amazon.com/images/I/31H4A9TbmFL.jpg\n",
      "5773 http://ecx.images-amazon.com/images/I/41k76LvBhNL.jpg\n",
      "5797 http://ecx.images-amazon.com/images/I/31bG9j9gspL.jpg\n",
      "5812 http://ecx.images-amazon.com/images/I/21IWFU2l7iL.jpg\n",
      "5813 http://ecx.images-amazon.com/images/I/21XG70A4GPL.jpg\n",
      "5815 http://ecx.images-amazon.com/images/I/21tpCQjUI%2BL.jpg\n",
      "5844 http://ecx.images-amazon.com/images/I/21qYWhuC4CL.jpg\n",
      "5848 http://ecx.images-amazon.com/images/I/21AcapzpPgL.jpg\n",
      "5889 http://ecx.images-amazon.com/images/I/31ito2apwVL.jpg\n",
      "5910 http://ecx.images-amazon.com/images/I/41I7bSu%2BfpL.jpg\n",
      "5924 http://ecx.images-amazon.com/images/I/31qcAh7P0WL.jpg\n",
      "5970 http://ecx.images-amazon.com/images/I/31cyogwaiyL.jpg\n",
      "6033 http://ecx.images-amazon.com/images/I/31yyevfdktL.jpg\n",
      "6035 http://ecx.images-amazon.com/images/I/313-ZLW%2BeFL.jpg\n",
      "6043 http://ecx.images-amazon.com/images/I/217rLxoBa7L.jpg\n",
      "6064 http://ecx.images-amazon.com/images/I/31ghoAhtZTL.jpg\n",
      "6161 http://ecx.images-amazon.com/images/I/31Dxu8Rb%2BsL.jpg\n",
      "6177 http://ecx.images-amazon.com/images/I/31LCkGmhEhL.jpg\n",
      "6179 http://ecx.images-amazon.com/images/I/31mOWF3cQrL.jpg\n",
      "6182 http://ecx.images-amazon.com/images/I/311MkJPiAPL.jpg\n",
      "6186 http://ecx.images-amazon.com/images/I/21Ds82lQRQL.jpg\n",
      "6204 http://ecx.images-amazon.com/images/I/11xAd2aVmdL.jpg\n",
      "6221 http://ecx.images-amazon.com/images/I/21PzDoiALsL.jpg\n",
      "6358 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "6390 http://ecx.images-amazon.com/images/I/41CbaiOh6cL._SX342_.jpg\n",
      "6399 http://ecx.images-amazon.com/images/I/31%2B2JbA66eL.jpg\n",
      "6402 http://ecx.images-amazon.com/images/I/31JBu-TOwKL.jpg\n",
      "6424 http://ecx.images-amazon.com/images/I/11Hg3v7oRDL.jpg\n",
      "6448 http://ecx.images-amazon.com/images/I/31roSEiH1GL.jpg\n",
      "6461 http://ecx.images-amazon.com/images/I/31AMUF6DDEL.jpg\n",
      "6462 http://ecx.images-amazon.com/images/I/21wsH%2BBwF6L.jpg\n",
      "6464 http://ecx.images-amazon.com/images/I/21rEEqPLasL.jpg\n",
      "6467 http://ecx.images-amazon.com/images/I/3157-XIzkFL.jpg\n",
      "6468 http://ecx.images-amazon.com/images/I/31yCzURX-AL.jpg\n",
      "6476 http://ecx.images-amazon.com/images/I/31I6Sv4XRdL.jpg\n",
      "6488 http://ecx.images-amazon.com/images/I/31j4rfD5ySL.jpg\n",
      "6498 http://ecx.images-amazon.com/images/I/21EFNTHN87L.jpg\n",
      "6513 http://ecx.images-amazon.com/images/I/21YA7WWxX2L.jpg\n",
      "6519 http://ecx.images-amazon.com/images/I/31d2yPYej4L._SX342_.jpg\n",
      "6529 http://ecx.images-amazon.com/images/I/21PPhdjPuAL.jpg\n",
      "6551 http://ecx.images-amazon.com/images/I/31a%2BpCjmbdL.jpg\n",
      "6575 http://ecx.images-amazon.com/images/I/11SMAiOSnOL.jpg\n",
      "6577 http://ecx.images-amazon.com/images/I/21aPqxLkUYL.jpg\n",
      "6586 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "6615 http://ecx.images-amazon.com/images/I/216ODx1tEAL.jpg\n",
      "6621 http://ecx.images-amazon.com/images/I/21M-cWXsFxL.jpg\n",
      "6624 http://ecx.images-amazon.com/images/I/31wsj549x1L.jpg\n",
      "6630 http://ecx.images-amazon.com/images/I/31GTs%2BZ2-0L.jpg\n",
      "6705 http://ecx.images-amazon.com/images/I/316YrHUGaNL.jpg\n",
      "6731 http://ecx.images-amazon.com/images/I/1115al4EckL.jpg\n",
      "6737 http://ecx.images-amazon.com/images/I/31UD%2BrrKY%2BL.jpg\n",
      "6768 http://ecx.images-amazon.com/images/I/215QIbDb%2BqL.jpg\n",
      "6774 http://ecx.images-amazon.com/images/I/31ClcIflYSL.jpg\n",
      "6775 http://ecx.images-amazon.com/images/I/41h5Bk85dNL.jpg\n",
      "6813 http://ecx.images-amazon.com/images/I/2187ayG9MEL.jpg\n",
      "6827 http://ecx.images-amazon.com/images/I/21Y7-QNls3L.jpg\n",
      "6845 http://ecx.images-amazon.com/images/I/411jo-OUGeL.jpg\n",
      "6891 http://ecx.images-amazon.com/images/I/318mb-vOCPL.jpg\n",
      "6937 http://ecx.images-amazon.com/images/I/31fcx7I-a8L.jpg\n",
      "6981 http://ecx.images-amazon.com/images/I/11gWtKRsSDL.jpg\n",
      "6984 http://ecx.images-amazon.com/images/I/31pSJBnfMrL.jpg\n",
      "6991 http://ecx.images-amazon.com/images/I/21CBKjYbbGL.jpg\n",
      "7017 http://ecx.images-amazon.com/images/I/11ArRWnC%2BSL.jpg\n",
      "7057 http://ecx.images-amazon.com/images/I/41e6uuK026L.jpg\n",
      "7103 http://ecx.images-amazon.com/images/I/21awg-yCOzL.jpg\n",
      "7126 http://ecx.images-amazon.com/images/I/21QcsVTg12L.jpg\n",
      "7159 http://ecx.images-amazon.com/images/I/415XtSUrL%2BL.jpg\n",
      "7168 http://ecx.images-amazon.com/images/I/41RpDUZTvUL.jpg\n",
      "7172 http://ecx.images-amazon.com/images/I/318DhJYcFyL.jpg\n",
      "7197 http://ecx.images-amazon.com/images/I/31hy5AqMw5L.jpg\n",
      "7198 http://ecx.images-amazon.com/images/I/11J47-ow5%2BL.jpg\n",
      "7224 http://ecx.images-amazon.com/images/I/418C949o6sL._SX342_.jpg\n",
      "7242 http://ecx.images-amazon.com/images/I/51juLA1UbAL._SX395_.jpg\n",
      "7248 http://ecx.images-amazon.com/images/I/11IuMX64xsL.jpg\n",
      "7255 http://ecx.images-amazon.com/images/I/316%2BRBQ1zJL.jpg\n",
      "7261 http://ecx.images-amazon.com/images/I/21uhOgmEC4L.jpg\n",
      "7263 http://ecx.images-amazon.com/images/I/11%2BOsSlZSvL.jpg\n",
      "7269 http://ecx.images-amazon.com/images/I/41l5rNF%2B22L.jpg\n",
      "7279 http://ecx.images-amazon.com/images/I/21wVhcRu1HL.jpg\n",
      "7287 http://ecx.images-amazon.com/images/I/41qId5uuaBL.jpg\n",
      "7318 http://ecx.images-amazon.com/images/I/313MgCMvYvL.jpg\n",
      "7342 http://ecx.images-amazon.com/images/I/21TAy1m1JbL.jpg\n",
      "7360 http://ecx.images-amazon.com/images/I/4110DKPcbeL._SY395_.jpg\n",
      "7383 http://ecx.images-amazon.com/images/I/21bJULo6DfL.jpg\n",
      "7448 http://ecx.images-amazon.com/images/I/41OgjspfLxL.jpg\n",
      "7464 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "7476 http://ecx.images-amazon.com/images/I/41O-IBdzeKL.jpg\n",
      "7477 http://ecx.images-amazon.com/images/I/31ym3d50nyL.jpg\n",
      "7489 http://ecx.images-amazon.com/images/I/41BIZC0oBmL.jpg\n",
      "7490 http://ecx.images-amazon.com/images/I/41BOrToKzhL.jpg\n",
      "7491 http://ecx.images-amazon.com/images/I/41zJJE4NiLL.jpg\n",
      "7494 http://ecx.images-amazon.com/images/I/31xBlpM49yL.jpg\n",
      "7501 http://ecx.images-amazon.com/images/I/11LoeWfvraL.jpg\n",
      "7510 http://ecx.images-amazon.com/images/I/11404ILxMPL.jpg\n",
      "7567 http://ecx.images-amazon.com/images/I/315sH2chKWL.jpg\n",
      "7574 http://ecx.images-amazon.com/images/I/31KYKgrH0JL.jpg\n",
      "7581 http://ecx.images-amazon.com/images/I/21CsemWvYrL.jpg\n",
      "7608 http://ecx.images-amazon.com/images/I/111rxJ54JlL.jpg\n",
      "7614 http://ecx.images-amazon.com/images/I/21R5N5iyNlL.jpg\n",
      "7642 http://ecx.images-amazon.com/images/I/41olvQM60cL.jpg\n",
      "7654 http://ecx.images-amazon.com/images/I/21Mtvrnx0dL.jpg\n",
      "7675 http://ecx.images-amazon.com/images/I/31u-e4LLNOL.jpg\n",
      "7689 http://ecx.images-amazon.com/images/I/31tCiHHJyCL.jpg\n",
      "7722 http://ecx.images-amazon.com/images/I/31dhhaQbU9L.jpg\n",
      "7730 http://ecx.images-amazon.com/images/I/313RJg7rONL.jpg\n",
      "7734 http://ecx.images-amazon.com/images/I/31MI%2BHPv8JL.jpg\n",
      "7749 http://ecx.images-amazon.com/images/I/41ekaZH7%2BhL.jpg\n",
      "7784 http://ecx.images-amazon.com/images/I/31efZHJdK8L.jpg\n",
      "7821 http://ecx.images-amazon.com/images/I/41gIAB3to8L.jpg\n",
      "7888 http://ecx.images-amazon.com/images/I/41ahNpBeyVL.jpg\n",
      "7909 http://ecx.images-amazon.com/images/I/21yFeTBv0RL.jpg\n",
      "7951 http://ecx.images-amazon.com/images/I/31KmSMm6CKL.jpg\n",
      "7963 http://ecx.images-amazon.com/images/I/31AeLPyi4lL.jpg\n",
      "7969 http://ecx.images-amazon.com/images/I/21GI2U0NlDL.jpg\n",
      "7982 http://ecx.images-amazon.com/images/I/31oMX7qOKIL.jpg\n",
      "7993 http://ecx.images-amazon.com/images/I/31m70EOwYkL.jpg\n",
      "8009 http://ecx.images-amazon.com/images/I/21sh0Eezj4L.jpg\n",
      "8014 http://ecx.images-amazon.com/images/I/41Lx5quQT%2BL.jpg\n",
      "8015 http://ecx.images-amazon.com/images/I/31H1kw9o6cL.jpg\n",
      "8046 http://ecx.images-amazon.com/images/I/31mtSndmTTL.jpg\n",
      "8068 http://ecx.images-amazon.com/images/I/21QVUqq1v4L.jpg\n",
      "8069 http://ecx.images-amazon.com/images/I/117Lf%2BEC1tL.jpg\n",
      "8070 http://ecx.images-amazon.com/images/I/21k1MlrNOjL.jpg\n",
      "8071 http://ecx.images-amazon.com/images/I/21AnozY9vzL.jpg\n",
      "8086 http://ecx.images-amazon.com/images/I/31V3gGDDipL.jpg\n",
      "8088 http://ecx.images-amazon.com/images/I/21H7yBW6keL.jpg\n",
      "8093 http://ecx.images-amazon.com/images/I/21LBU8jpGKL.jpg\n",
      "8104 http://ecx.images-amazon.com/images/I/21ax4rnLYnL.jpg\n",
      "8117 http://ecx.images-amazon.com/images/I/41MCLuHv3xL.jpg\n",
      "8129 http://ecx.images-amazon.com/images/I/31xX1HfFTqL.jpg\n",
      "8132 http://ecx.images-amazon.com/images/I/31wYrKKDffL.jpg\n",
      "8136 http://ecx.images-amazon.com/images/I/31oDyXXt7IL.jpg\n",
      "8157 http://ecx.images-amazon.com/images/I/11zyj%2BYbUwL.jpg\n",
      "8165 http://ecx.images-amazon.com/images/I/511e1RLgsYL._SY395_.jpg\n",
      "8178 http://ecx.images-amazon.com/images/I/21llm82l8BL.jpg\n",
      "8179 http://ecx.images-amazon.com/images/I/314fC34H9pL.jpg\n",
      "8211 http://ecx.images-amazon.com/images/I/215uasK3lfL.jpg\n",
      "8215 http://ecx.images-amazon.com/images/I/21ApRWCOLmL.jpg\n",
      "8243 http://ecx.images-amazon.com/images/I/217WZXYUjOL.jpg\n",
      "8256 http://ecx.images-amazon.com/images/I/21UdRXXRBPL.jpg\n",
      "8318 http://ecx.images-amazon.com/images/I/31PhT%2B7sfKL.jpg\n",
      "8319 http://ecx.images-amazon.com/images/I/313kZhLaWyL.jpg\n",
      "8337 http://ecx.images-amazon.com/images/I/21PspyJDaXL.jpg\n",
      "8344 http://ecx.images-amazon.com/images/I/21mmakHMcvL.jpg\n",
      "8355 http://ecx.images-amazon.com/images/I/31v471lV%2BsL.jpg\n",
      "8398 http://ecx.images-amazon.com/images/I/21mq8EC18hL.jpg\n",
      "8400 http://ecx.images-amazon.com/images/I/11dBWf2l2EL.jpg\n",
      "8404 http://ecx.images-amazon.com/images/I/21COI-r8gRL.jpg\n",
      "8484 http://ecx.images-amazon.com/images/I/21fTi1qbDxL.jpg\n",
      "8518 http://ecx.images-amazon.com/images/I/21YMsNYFejL.jpg\n",
      "8522 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "8523 http://ecx.images-amazon.com/images/I/11Ovt1vzB2L.jpg\n",
      "8534 http://ecx.images-amazon.com/images/I/2136SGCYxvL.jpg\n",
      "8581 http://ecx.images-amazon.com/images/I/31nt2bmTIcL.jpg\n",
      "8619 http://ecx.images-amazon.com/images/I/315JChRcvzL.jpg\n",
      "8624 http://ecx.images-amazon.com/images/I/31nly6Dn5DL.jpg\n",
      "8656 http://ecx.images-amazon.com/images/I/317tf0yspOL.jpg\n",
      "8663 http://ecx.images-amazon.com/images/I/11Q%2BGanjUDL.jpg\n",
      "8670 http://ecx.images-amazon.com/images/I/21PhodVPeiL.jpg\n",
      "8684 http://ecx.images-amazon.com/images/I/41AmsSqmFML.jpg\n",
      "8696 http://ecx.images-amazon.com/images/I/21iBFIs7XwL.jpg\n",
      "8709 http://ecx.images-amazon.com/images/I/31Qiu4URQ8L.jpg\n",
      "8744 http://ecx.images-amazon.com/images/I/31zp0BC4IdL.jpg\n",
      "8745 http://ecx.images-amazon.com/images/I/31zp0BC4IdL.jpg\n",
      "8763 http://ecx.images-amazon.com/images/I/219Vjumb5KL.jpg\n",
      "8766 http://ecx.images-amazon.com/images/I/01mSuFCZ29L.jpg\n",
      "8788 http://ecx.images-amazon.com/images/I/31wjfNMXcaL.jpg\n",
      "8821 http://ecx.images-amazon.com/images/I/311%2B9mJ-8ZL.jpg\n",
      "8853 http://ecx.images-amazon.com/images/I/319ao5n-XaL.jpg\n",
      "8861 http://ecx.images-amazon.com/images/I/31E%2Bd-8ChOL.jpg\n",
      "8959 http://ecx.images-amazon.com/images/I/51862fSXi5L._SX342_.jpg\n",
      "8973 http://ecx.images-amazon.com/images/I/21zz%2BNMs0aL.jpg\n",
      "8987 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "8993 http://ecx.images-amazon.com/images/I/41Jq%2BsPco6L.jpg\n",
      "8994 http://ecx.images-amazon.com/images/I/31snZRj2jJL.jpg\n",
      "8995 http://ecx.images-amazon.com/images/I/41VC6WCuM%2BL.jpg\n",
      "9100 http://ecx.images-amazon.com/images/I/315GrzPrUqL.jpg\n",
      "9138 http://ecx.images-amazon.com/images/I/210yS4-0W2L.jpg\n",
      "9148 http://ecx.images-amazon.com/images/I/21DU7phFwFL.jpg\n",
      "9173 http://ecx.images-amazon.com/images/I/21tFhi0gt6L.jpg\n",
      "9175 http://ecx.images-amazon.com/images/I/21iwtyVblNL.jpg\n",
      "9268 http://ecx.images-amazon.com/images/I/31S2d7iso%2BL.jpg\n",
      "9291 http://ecx.images-amazon.com/images/I/3125jVFVvGL.jpg\n",
      "9294 http://ecx.images-amazon.com/images/I/31HUGv7pjmL.jpg\n",
      "9303 http://ecx.images-amazon.com/images/I/21zMf-J2ihL.jpg\n",
      "9339 http://ecx.images-amazon.com/images/I/31L20kBXp-L.jpg\n",
      "9401 http://ecx.images-amazon.com/images/I/21J8VkszTHL.jpg\n",
      "9419 http://ecx.images-amazon.com/images/I/51JsRaCpl1L._SY395_.jpg\n",
      "9448 http://ecx.images-amazon.com/images/I/31UqUQ-vZCL.jpg\n",
      "9477 http://ecx.images-amazon.com/images/I/41syGohf-NL.jpg\n",
      "9478 http://ecx.images-amazon.com/images/I/41f0q4evMxL.jpg\n",
      "9498 http://ecx.images-amazon.com/images/I/41It3iWH-eL.jpg\n",
      "9499 http://ecx.images-amazon.com/images/I/31p4-poKrrL.jpg\n",
      "9524 http://ecx.images-amazon.com/images/I/21AErwVE-IL.jpg\n",
      "9564 http://ecx.images-amazon.com/images/I/21U0LHpuhGL.jpg\n",
      "9579 http://ecx.images-amazon.com/images/I/312kxMAvxBL.jpg\n",
      "9600 http://ecx.images-amazon.com/images/I/21WdC92INqL.jpg\n",
      "9604 http://ecx.images-amazon.com/images/I/31XIfFsHouL.jpg\n",
      "9640 http://ecx.images-amazon.com/images/I/311PQ1EaYDL.jpg\n",
      "9650 http://ecx.images-amazon.com/images/I/31DkxVUcrtL.jpg\n",
      "9676 http://ecx.images-amazon.com/images/I/41XhpCUw9NL.jpg\n",
      "9680 http://ecx.images-amazon.com/images/I/11dau4x%2B%2B2L.jpg\n",
      "9704 http://ecx.images-amazon.com/images/I/21KDAGrSB0L.jpg\n",
      "9713 http://ecx.images-amazon.com/images/I/31nJFeWg-0L.jpg\n",
      "9730 http://ecx.images-amazon.com/images/I/41aKNIR%2Bc4L._SX342_.jpg\n",
      "9737 http://ecx.images-amazon.com/images/I/21vZBlIZsWL.jpg\n",
      "9771 http://ecx.images-amazon.com/images/I/21W5Z%2BAQQhL.jpg\n",
      "9792 http://ecx.images-amazon.com/images/I/21wW-fEEhsL.jpg\n",
      "9866 http://ecx.images-amazon.com/images/I/113ZEaEMn9L.jpg\n",
      "9878 http://ecx.images-amazon.com/images/I/31lMF6qcM6L.jpg\n",
      "9904 http://ecx.images-amazon.com/images/I/31z5-UsYoPL.jpg\n",
      "9932 http://ecx.images-amazon.com/images/I/21QtwzvxcaL.jpg\n",
      "9955 http://ecx.images-amazon.com/images/I/31f1i-KHyFL.jpg\n",
      "9964 http://ecx.images-amazon.com/images/I/41Dvi-iPJSL.jpg\n",
      "9996 http://ecx.images-amazon.com/images/I/31CGWNrGdfL._SY445_.jpg\n",
      "10021 http://ecx.images-amazon.com/images/I/416tSO7avpL.jpg\n",
      "10027 http://ecx.images-amazon.com/images/I/31nEPkcvsoL.jpg\n",
      "10044 http://ecx.images-amazon.com/images/I/41hee0tH3UL.jpg\n",
      "10049 http://ecx.images-amazon.com/images/I/11enNJB1qiL.jpg\n",
      "10052 http://ecx.images-amazon.com/images/I/21Dqc5OwZVL.jpg\n",
      "10099 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "10114 http://ecx.images-amazon.com/images/I/31cL2-QFDRL._SX342_.jpg\n",
      "10143 http://ecx.images-amazon.com/images/I/21Gd20T4o9L.jpg\n",
      "10181 http://ecx.images-amazon.com/images/I/218c764bXkL.jpg\n",
      "10201 http://ecx.images-amazon.com/images/I/21NgH6ZNT3L.jpg\n",
      "10279 http://ecx.images-amazon.com/images/I/21JdDP5jsZL.jpg\n",
      "10305 http://ecx.images-amazon.com/images/I/31jEICFHbgL.jpg\n",
      "10312 http://ecx.images-amazon.com/images/I/31JB9HAxkNL.jpg\n",
      "10323 http://ecx.images-amazon.com/images/I/21lcuhoosWL.jpg\n",
      "10346 http://ecx.images-amazon.com/images/I/118uMZOYkzL.jpg\n",
      "10380 http://ecx.images-amazon.com/images/I/21fqzZaJUdL.jpg\n",
      "10403 http://ecx.images-amazon.com/images/I/31mstFieTVL.jpg\n",
      "10410 http://ecx.images-amazon.com/images/I/11RWEtFa%2BpL.jpg\n",
      "10429 http://ecx.images-amazon.com/images/I/31tTb0fZdYL.jpg\n",
      "10495 http://ecx.images-amazon.com/images/I/21INBW1BI2L.jpg\n",
      "10536 http://ecx.images-amazon.com/images/I/314IwVkR0ML.jpg\n",
      "10640 http://ecx.images-amazon.com/images/I/41lXtGAaMHL._SX342_.jpg\n",
      "10650 http://ecx.images-amazon.com/images/I/5188HCZ2Z5L._SX342_.jpg\n",
      "10651 http://ecx.images-amazon.com/images/I/41l9vWfqN3L.jpg\n",
      "10678 http://ecx.images-amazon.com/images/I/31DYaJZClRL.jpg\n",
      "10755 http://ecx.images-amazon.com/images/I/41SkgoAgZnL.jpg\n",
      "10760 http://ecx.images-amazon.com/images/I/41NW7cuJLPL.jpg\n",
      "10801 http://ecx.images-amazon.com/images/I/41ZROzzpArL.jpg\n",
      "10802 http://ecx.images-amazon.com/images/I/41CSamhp-GL.jpg\n",
      "10969 http://ecx.images-amazon.com/images/I/31ig5Q1G4AL.jpg\n",
      "10992 http://ecx.images-amazon.com/images/I/41F3eziuLOL.jpg\n",
      "11004 http://ecx.images-amazon.com/images/I/41JH8MHGFzL.jpg\n",
      "11039 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "11041 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "11042 http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif\n",
      "11059 http://ecx.images-amazon.com/images/I/215%2BIibPyoL.jpg\n",
      "11066 http://ecx.images-amazon.com/images/I/41Tmrn4KW0L.jpg\n",
      "11098 http://ecx.images-amazon.com/images/I/31bcuE8NGoL.jpg\n",
      "11119 http://ecx.images-amazon.com/images/I/21pwflw5H6L.jpg\n",
      "11129 http://ecx.images-amazon.com/images/I/21B8j%2BnzEnL.jpg\n",
      "11207 http://ecx.images-amazon.com/images/I/21TAz6GSQ2L.jpg\n",
      "11294 http://ecx.images-amazon.com/images/I/219XE%2BQ6z-L.jpg\n",
      "11295 http://ecx.images-amazon.com/images/I/31NaI65MG9L.jpg\n",
      "11316 http://ecx.images-amazon.com/images/I/41xLs3bJ3-L.jpg\n",
      "11330 http://ecx.images-amazon.com/images/I/319-Hsu7t5L.jpg\n",
      "11342 http://ecx.images-amazon.com/images/I/21uCQ5G2iFL.jpg\n",
      "11383 http://ecx.images-amazon.com/images/I/51yvZUxMyiL._SX342_.jpg\n",
      "11482 http://ecx.images-amazon.com/images/I/41JB3IhBWXL.jpg\n",
      "11540 http://ecx.images-amazon.com/images/I/21vvmhE3C%2BL.jpg\n",
      "11541 http://ecx.images-amazon.com/images/I/318K%2BTVxmwL.jpg\n",
      "11553 http://ecx.images-amazon.com/images/I/31NtR6YkhqL.jpg\n",
      "11558 http://ecx.images-amazon.com/images/I/21FMYMIRRqL.jpg\n",
      "11635 http://ecx.images-amazon.com/images/I/11U7UqO57UL.jpg\n",
      "11646 http://ecx.images-amazon.com/images/I/41akVHnBE3L._SX342_.jpg\n",
      "11648 http://ecx.images-amazon.com/images/I/41%2BtinGOBbL.jpg\n",
      "11651 http://ecx.images-amazon.com/images/I/21DF2jtp%2BVL.jpg\n",
      "11724 http://ecx.images-amazon.com/images/I/31dZ7N-2fkL.jpg\n",
      "11820 http://ecx.images-amazon.com/images/I/51A0EWU%2BWmL._SX342_.jpg\n",
      "11834 http://ecx.images-amazon.com/images/I/21r6ll8Sa1L.jpg\n",
      "11906 http://ecx.images-amazon.com/images/I/31bTV3zveUL.jpg\n",
      "NaN:7,{'._SX300_.': 1865, '._SY300_.': 9581, 'no_suffix': 648}:12094,suffix_sum:12094,item_n:12101\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# pattern = re.compile('(\\._).*(_\\.)',re.S)\n",
    "# suffix=pattern.findall(row['imUrl'])\n",
    "\n",
    "\n",
    "img_list=[]\n",
    "count_NaN=0\n",
    "count_suffix=0\n",
    "\n",
    "resolution_suffix= {'._SX300_.':0,'._SY300_.':0,'no_suffix':0}\n",
    "for i,row in i_meta_df.iterrows():\n",
    "    # print(type(row['imUrl']))\n",
    "    if isinstance(row['imUrl'],float):\n",
    "        count_NaN+=1\n",
    "        continue\n",
    "    if isinstance(row['imUrl'],str):\n",
    "\n",
    "        if '._SX300_.' in row['imUrl']:\n",
    "            resolution_suffix['._SX300_.']+=1\n",
    "        elif '._SY300_.' in row['imUrl']:\n",
    "            resolution_suffix['._SY300_.']+=1\n",
    "        else:\n",
    "            print(i,row['imUrl'])\n",
    "            resolution_suffix['no_suffix']+=1\n",
    "        count_suffix+=1\n",
    "\n",
    "print(f\"NaN:{count_NaN},{resolution_suffix}:{count_suffix},suffix_sum:{sum([resolution_suffix[i] for i in resolution_suffix])},item_n:{len(i_meta_df)}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration: 50 \t\ttime:19.34s\n",
      "iteration: 100 \t\ttime:17.55s\n",
      "iteration: 150 \t\ttime:16.61s\n",
      "iteration: 200 \t\ttime:23.23s\n",
      "iteration: 250 \t\ttime:25.91s\n",
      "iteration: 300 \t\ttime:25.11s\n",
      "iteration: 350 \t\ttime:26.79s\n",
      "iteration: 400 \t\ttime:25.12s\n",
      "iteration: 450 \t\ttime:25.86s\n",
      "iteration: 500 \t\ttime:25.64s\n",
      "iteration: 550 \t\ttime:25.31s\n",
      "iteration: 600 \t\ttime:24.81s\n",
      "iteration: 650 \t\ttime:24.80s\n",
      "iteration: 700 \t\ttime:25.08s\n",
      "725:http://ecx.images-amazon.com/images/I/41jAeco8PWL._SY300_.jpg is a invalid url!\n",
      "727:http://ecx.images-amazon.com/images/I/31P3XECD3FL._SY300_.jpg is a invalid url!\n",
      "iteration: 750 \t\ttime:25.62s\n",
      "iteration: 800 \t\ttime:24.11s\n",
      "834:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 850 \t\ttime:25.04s\n",
      "iteration: 900 \t\ttime:24.98s\n",
      "iteration: 950 \t\ttime:27.30s\n",
      "iteration: 1000 \t\ttime:24.03s\n",
      "1046:http://ecx.images-amazon.com/images/I/21x%2BHtGtwyL._SY300_.jpg is a invalid url!\n",
      "iteration: 1050 \t\ttime:25.78s\n",
      "iteration: 1100 \t\ttime:24.87s\n",
      "iteration: 1150 \t\ttime:25.17s\n",
      "1168:http://ecx.images-amazon.com/images/I/41eBfqU2mrL._SY300_.jpg is a invalid url!\n",
      "iteration: 1200 \t\ttime:24.74s\n",
      "iteration: 1250 \t\ttime:24.15s\n",
      "iteration: 1300 \t\ttime:25.06s\n",
      "1346:http://ecx.images-amazon.com/images/I/41EARXKK40L.gif is a invalid url!\n",
      "iteration: 1350 \t\ttime:25.79s\n",
      "iteration: 1400 \t\ttime:25.90s\n",
      "1411:http://ecx.images-amazon.com/images/I/31dBeyI%2BohL._SY300_.jpg is a invalid url!\n",
      "iteration: 1450 \t\ttime:155.72s\n",
      "1460 is nan\n",
      "iteration: 1500 \t\ttime:24.33s\n",
      "iteration: 1550 \t\ttime:24.98s\n",
      "iteration: 1600 \t\ttime:25.51s\n",
      "iteration: 1650 \t\ttime:26.33s\n",
      "iteration: 1700 \t\ttime:25.92s\n",
      "iteration: 1750 \t\ttime:25.19s\n",
      "iteration: 1800 \t\ttime:25.64s\n",
      "iteration: 1850 \t\ttime:25.87s\n",
      "1873:http://ecx.images-amazon.com/images/I/61pNsVQxpjL.gif is a invalid url!\n",
      "1890:http://ecx.images-amazon.com/images/I/31irx0%2Bww5L._SY300_.jpg is a invalid url!\n",
      "iteration: 1900 \t\ttime:155.20s\n",
      "1919:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 1950 \t\ttime:26.39s\n",
      "iteration: 2000 \t\ttime:24.60s\n",
      "iteration: 2050 \t\ttime:27.31s\n",
      "iteration: 2100 \t\ttime:27.02s\n",
      "iteration: 2150 \t\ttime:25.78s\n",
      "iteration: 2200 \t\ttime:26.47s\n",
      "iteration: 2250 \t\ttime:25.06s\n",
      "iteration: 2300 \t\ttime:25.33s\n",
      "iteration: 2350 \t\ttime:25.43s\n",
      "iteration: 2400 \t\ttime:23.51s\n",
      "iteration: 2450 \t\ttime:23.88s\n",
      "iteration: 2500 \t\ttime:24.29s\n",
      "iteration: 2550 \t\ttime:25.91s\n",
      "iteration: 2600 \t\ttime:25.50s\n",
      "iteration: 2650 \t\ttime:25.27s\n",
      "iteration: 2700 \t\ttime:26.80s\n",
      "iteration: 2750 \t\ttime:25.19s\n",
      "iteration: 2800 \t\ttime:24.57s\n",
      "iteration: 2850 \t\ttime:26.20s\n",
      "iteration: 2900 \t\ttime:24.94s\n",
      "iteration: 2950 \t\ttime:24.93s\n",
      "2987:http://ecx.images-amazon.com/images/I/51UmJ85cLWL.gif is a invalid url!\n",
      "iteration: 3000 \t\ttime:26.88s\n",
      "iteration: 3050 \t\ttime:25.40s\n",
      "iteration: 3100 \t\ttime:24.09s\n",
      "iteration: 3150 \t\ttime:26.44s\n",
      "iteration: 3200 \t\ttime:24.14s\n",
      "iteration: 3250 \t\ttime:24.63s\n",
      "iteration: 3300 \t\ttime:26.34s\n",
      "3338:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 3350 \t\ttime:25.00s\n",
      "3382:http://ecx.images-amazon.com/images/I/31dqZKOBVDL._SY300_.jpg is a invalid url!\n",
      "iteration: 3400 \t\ttime:154.43s\n",
      "iteration: 3450 \t\ttime:25.58s\n",
      "iteration: 3500 \t\ttime:24.67s\n",
      "iteration: 3550 \t\ttime:25.64s\n",
      "3578 is nan\n",
      "iteration: 3600 \t\ttime:24.76s\n",
      "iteration: 3650 \t\ttime:26.12s\n",
      "iteration: 3700 \t\ttime:24.34s\n",
      "iteration: 3750 \t\ttime:25.29s\n",
      "iteration: 3800 \t\ttime:25.22s\n",
      "iteration: 3850 \t\ttime:25.28s\n",
      "3860:http://ecx.images-amazon.com/images/I/41YWR7V0VQL._SY300_.jpg is a invalid url!\n",
      "iteration: 3900 \t\ttime:154.21s\n",
      "iteration: 3950 \t\ttime:24.69s\n",
      "iteration: 4000 \t\ttime:25.55s\n",
      "iteration: 4050 \t\ttime:24.30s\n",
      "iteration: 4100 \t\ttime:26.09s\n",
      "4106:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 4150 \t\ttime:25.69s\n",
      "iteration: 4200 \t\ttime:25.50s\n",
      "4208 is nan\n",
      "iteration: 4250 \t\ttime:23.52s\n",
      "iteration: 4300 \t\ttime:25.27s\n",
      "iteration: 4350 \t\ttime:24.84s\n",
      "4381 is nan\n",
      "4398 is nan\n",
      "iteration: 4400 \t\ttime:24.34s\n",
      "iteration: 4450 \t\ttime:26.50s\n",
      "iteration: 4500 \t\ttime:26.31s\n",
      "iteration: 4550 \t\ttime:26.49s\n",
      "iteration: 4600 \t\ttime:25.06s\n",
      "iteration: 4650 \t\ttime:24.60s\n",
      "iteration: 4700 \t\ttime:25.58s\n",
      "iteration: 4750 \t\ttime:24.71s\n",
      "iteration: 4800 \t\ttime:25.83s\n",
      "iteration: 4850 \t\ttime:26.70s\n",
      "iteration: 4900 \t\ttime:27.10s\n",
      "iteration: 4950 \t\ttime:24.80s\n",
      "4981:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 5000 \t\ttime:25.11s\n",
      "iteration: 5050 \t\ttime:25.14s\n",
      "iteration: 5100 \t\ttime:25.17s\n",
      "iteration: 5150 \t\ttime:25.56s\n",
      "iteration: 5200 \t\ttime:25.06s\n",
      "iteration: 5250 \t\ttime:27.27s\n",
      "iteration: 5300 \t\ttime:25.27s\n",
      "iteration: 5350 \t\ttime:27.35s\n",
      "5357:http://ecx.images-amazon.com/images/I/31lBHtZICyL._SY300_.jpg is a invalid url!\n",
      "iteration: 5400 \t\ttime:155.89s\n",
      "iteration: 5450 \t\ttime:26.17s\n",
      "iteration: 5500 \t\ttime:26.51s\n",
      "iteration: 5550 \t\ttime:26.15s\n",
      "iteration: 5600 \t\ttime:26.65s\n",
      "iteration: 5650 \t\ttime:27.04s\n",
      "iteration: 5700 \t\ttime:24.68s\n",
      "iteration: 5750 \t\ttime:25.97s\n",
      "iteration: 5800 \t\ttime:26.19s\n",
      "5830:http://ecx.images-amazon.com/images/I/41wED6wBf2L._SY300_.jpg is a invalid url!\n",
      "iteration: 5850 \t\ttime:154.69s\n",
      "iteration: 5900 \t\ttime:25.27s\n",
      "iteration: 5950 \t\ttime:26.84s\n",
      "iteration: 6000 \t\ttime:26.41s\n",
      "iteration: 6050 \t\ttime:24.69s\n",
      "iteration: 6100 \t\ttime:26.47s\n",
      "iteration: 6150 \t\ttime:26.14s\n",
      "iteration: 6200 \t\ttime:25.81s\n",
      "iteration: 6250 \t\ttime:26.45s\n",
      "iteration: 6300 \t\ttime:26.32s\n",
      "iteration: 6350 \t\ttime:25.76s\n",
      "6358:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 6400 \t\ttime:24.68s\n",
      "iteration: 6450 \t\ttime:26.38s\n",
      "iteration: 6500 \t\ttime:26.67s\n",
      "iteration: 6550 \t\ttime:27.03s\n",
      "6586:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 6600 \t\ttime:24.11s\n",
      "iteration: 6650 \t\ttime:25.22s\n",
      "iteration: 6700 \t\ttime:25.04s\n",
      "iteration: 6750 \t\ttime:24.94s\n",
      "iteration: 6800 \t\ttime:25.80s\n",
      "iteration: 6850 \t\ttime:24.35s\n",
      "iteration: 6900 \t\ttime:26.48s\n",
      "iteration: 6950 \t\ttime:24.21s\n",
      "iteration: 7000 \t\ttime:25.63s\n",
      "iteration: 7050 \t\ttime:25.20s\n",
      "iteration: 7100 \t\ttime:25.82s\n",
      "iteration: 7150 \t\ttime:25.92s\n",
      "iteration: 7200 \t\ttime:26.12s\n",
      "iteration: 7250 \t\ttime:25.79s\n",
      "iteration: 7300 \t\ttime:25.99s\n",
      "7320:http://ecx.images-amazon.com/images/I/41nFeFDM1kL._SY300_.jpg is a invalid url!\n",
      "iteration: 7350 \t\ttime:155.23s\n",
      "iteration: 7400 \t\ttime:24.38s\n",
      "iteration: 7450 \t\ttime:24.30s\n",
      "7464:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 7500 \t\ttime:25.99s\n",
      "iteration: 7550 \t\ttime:25.59s\n",
      "iteration: 7600 \t\ttime:25.55s\n",
      "iteration: 7650 \t\ttime:25.10s\n",
      "iteration: 7700 \t\ttime:25.75s\n",
      "iteration: 7750 \t\ttime:26.22s\n",
      "7798:http://ecx.images-amazon.com/images/I/41sv73FjGlL._SY300_.jpg is a invalid url!\n",
      "iteration: 7800 \t\ttime:155.48s\n",
      "iteration: 7850 \t\ttime:24.65s\n",
      "iteration: 7900 \t\ttime:24.93s\n",
      "iteration: 7950 \t\ttime:27.62s\n",
      "iteration: 8000 \t\ttime:25.95s\n",
      "iteration: 8050 \t\ttime:26.51s\n",
      "iteration: 8100 \t\ttime:25.86s\n",
      "iteration: 8150 \t\ttime:26.29s\n",
      "iteration: 8200 \t\ttime:26.45s\n",
      "iteration: 8250 \t\ttime:27.36s\n",
      "iteration: 8300 \t\ttime:26.64s\n",
      "iteration: 8350 \t\ttime:26.84s\n",
      "iteration: 8400 \t\ttime:25.80s\n",
      "iteration: 8450 \t\ttime:25.85s\n",
      "iteration: 8500 \t\ttime:25.56s\n",
      "8522:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 8550 \t\ttime:25.48s\n",
      "iteration: 8600 \t\ttime:26.84s\n",
      "iteration: 8650 \t\ttime:25.41s\n",
      "iteration: 8700 \t\ttime:26.45s\n",
      "iteration: 8750 \t\ttime:25.39s\n",
      "iteration: 8800 \t\ttime:25.55s\n",
      "iteration: 8850 \t\ttime:26.55s\n",
      "iteration: 8900 \t\ttime:26.12s\n",
      "iteration: 8950 \t\ttime:25.12s\n",
      "8987:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 9000 \t\ttime:26.97s\n",
      "iteration: 9050 \t\ttime:25.74s\n",
      "iteration: 9100 \t\ttime:26.45s\n",
      "iteration: 9150 \t\ttime:26.14s\n",
      "iteration: 9200 \t\ttime:25.10s\n",
      "iteration: 9250 \t\ttime:26.66s\n",
      "9289:http://ecx.images-amazon.com/images/I/41j2ceafMoL._SY300_.jpg is a invalid url!\n",
      "iteration: 9300 \t\ttime:156.02s\n",
      "iteration: 9350 \t\ttime:25.52s\n",
      "iteration: 9400 \t\ttime:27.06s\n",
      "iteration: 9450 \t\ttime:24.82s\n",
      "iteration: 9500 \t\ttime:30.55s\n",
      "iteration: 9550 \t\ttime:26.76s\n",
      "iteration: 9600 \t\ttime:26.44s\n",
      "iteration: 9650 \t\ttime:25.49s\n",
      "iteration: 9700 \t\ttime:26.56s\n",
      "iteration: 9750 \t\ttime:24.99s\n",
      "9767:http://ecx.images-amazon.com/images/I/3151DD-sJzL._SY300_.jpg is a invalid url!\n",
      "iteration: 9800 \t\ttime:155.64s\n",
      "iteration: 9850 \t\ttime:25.30s\n",
      "iteration: 9900 \t\ttime:33.07s\n",
      "iteration: 9950 \t\ttime:25.59s\n",
      "iteration: 10000 \t\ttime:24.78s\n",
      "iteration: 10050 \t\ttime:26.19s\n",
      "10099:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 10100 \t\ttime:26.33s\n",
      "iteration: 10150 \t\ttime:26.57s\n",
      "iteration: 10200 \t\ttime:26.28s\n",
      "iteration: 10250 \t\ttime:26.34s\n",
      "iteration: 10300 \t\ttime:26.57s\n",
      "iteration: 10350 \t\ttime:25.89s\n",
      "iteration: 10400 \t\ttime:25.94s\n",
      "iteration: 10450 \t\ttime:26.82s\n",
      "iteration: 10500 \t\ttime:27.11s\n",
      "iteration: 10550 \t\ttime:25.83s\n",
      "iteration: 10600 \t\ttime:26.45s\n",
      "iteration: 10650 \t\ttime:25.70s\n",
      "iteration: 10700 \t\ttime:24.70s\n",
      "iteration: 10750 \t\ttime:25.47s\n",
      "10771 is nan\n",
      "iteration: 10800 \t\ttime:26.14s\n",
      "iteration: 10850 \t\ttime:25.82s\n",
      "iteration: 10900 \t\ttime:27.63s\n",
      "iteration: 10950 \t\ttime:26.16s\n",
      "10990 is nan\n",
      "iteration: 11000 \t\ttime:24.89s\n",
      "11039:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "11041:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "11042:http://g-ecx.images-amazon.com/images/G/01/x-site/icons/no-img-sm._CB192198896_.gif is a invalid url!\n",
      "iteration: 11050 \t\ttime:24.05s\n",
      "iteration: 11100 \t\ttime:24.94s\n",
      "iteration: 11150 \t\ttime:26.34s\n",
      "iteration: 11200 \t\ttime:25.94s\n",
      "iteration: 11250 \t\ttime:25.84s\n",
      "11263:http://ecx.images-amazon.com/images/I/51gpwSBR26L._SY300_.jpg is a invalid url!\n",
      "iteration: 11300 \t\ttime:155.23s\n",
      "iteration: 11350 \t\ttime:26.48s\n",
      "iteration: 11400 \t\ttime:25.73s\n",
      "iteration: 11450 \t\ttime:26.59s\n",
      "iteration: 11500 \t\ttime:25.90s\n",
      "iteration: 11550 \t\ttime:27.58s\n",
      "iteration: 11600 \t\ttime:25.64s\n",
      "iteration: 11650 \t\ttime:26.12s\n",
      "iteration: 11700 \t\ttime:25.23s\n",
      "11739:http://ecx.images-amazon.com/images/I/51dcak4K9gL._SY300_.jpg is a invalid url!\n",
      "iteration: 11750 \t\ttime:155.79s\n",
      "iteration: 11800 \t\ttime:26.52s\n",
      "iteration: 11850 \t\ttime:25.99s\n",
      "iteration: 11900 \t\ttime:25.35s\n",
      "iteration: 11950 \t\ttime:25.74s\n",
      "iteration: 12000 \t\ttime:25.75s\n",
      "iteration: 12050 \t\ttime:26.00s\n",
      "iteration: 12100 \t\ttime:24.63s\n",
      "abstract time ends.Total time:7755.35s\n",
      "torch.Size([12101, 512])\n",
      "invalid image url count:33\n",
      "success!\n"
     ]
    }
   ],
   "source": [
    "#nan unvalid ---------------------=======features Mean\n",
    "\n",
    "\n",
    "from PIL import Image\n",
    "import requests\n",
    "from transformers import AutoProcessor, CLIPModel\n",
    "import  time\n",
    "\n",
    "model = CLIPModel.from_pretrained(\"./clip-vit-base-patch32\")\n",
    "processor = AutoProcessor.from_pretrained(\"./clip-vit-base-patch32\")\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "DATASET=\"Beauty\"\n",
    "RAW_PATH=os.path.join('./',DATASET)\n",
    "Image_PATH=os.path.join(RAW_PATH,'images')\n",
    "\n",
    "img_nan_id=[]\n",
    "img_bad_id=[]\n",
    "bad_img_count=0\n",
    "\n",
    "img_features=[]\n",
    "\n",
    "if not os.path.isdir(Image_PATH):\n",
    "    os.mkdir(Image_PATH)\n",
    "\n",
    "start=time.time()\n",
    "\n",
    "for i,row in i_meta_df.iterrows():\n",
    "    if i%50==0:\n",
    "        if i==0:\n",
    "            t1=time.time()\n",
    "        else:\n",
    "            print(\"iteration:\",i,\"\\t\\ttime:{:.2f}s\".format(time.time()-t1))\n",
    "            t1=time.time()\n",
    "    if row['imUrl'] is np.nan:\n",
    "        print(f\"{row['item_id']} is nan\")\n",
    "        img_nan_id.append(row['item_id'])\n",
    "        img_features.append(torch.zeros(1,512).to(device))\n",
    "    else:\n",
    "        try:\n",
    "            image=Image.open(requests.get(row['imUrl'],stream=True).raw)\n",
    "            image.save(os.path.join(Image_PATH,str(row['item_id']))+\".jpg\")\n",
    "            inputs = processor(images=image, return_tensors=\"pt\")\n",
    "            inputs.to(device)\n",
    "            with torch.no_grad():\n",
    "                img_ft = model.get_image_features(**inputs)\n",
    "            img_features.append(img_ft)\n",
    "        except:\n",
    "            print(f'{row[\"item_id\"]}:{row[\"imUrl\"]} is a invalid url!')\n",
    "            img_bad_id.append(row['item_id'])\n",
    "            bad_img_count+=1\n",
    "            img_features.append(torch.zeros(1,512).to(device))\n",
    "\n",
    "print(\"abstract time ends.Total time:{:.2f}s\".format(time.time()-start))\n",
    "\n",
    "pt_img_features=torch.stack(img_features)\n",
    "pt_img_features=pt_img_features.squeeze()\n",
    "pt_img_mean=torch.mean(pt_img_features,dim=0)\n",
    "print(pt_img_features.shape)\n",
    "\n",
    "for i in img_nan_id:\n",
    "    pt_img_features[i]=pt_img_mean\n",
    "\n",
    "for i in img_bad_id:\n",
    "    pt_img_features[i]=pt_img_mean\n",
    "\n",
    "\n",
    "print(f\"invalid image url count:{bad_img_count}\")\n",
    "\n",
    "torch.save(pt_img_features.to(\"cpu\"),os.path.join(RAW_PATH,\"img_feat.pt\"))\n",
    "print(\"success!\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Test feature match!"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [],
   "source": [
    "def test_image(url,n):\n",
    "    image=Image.open(requests.get(url,stream=True).raw)\n",
    "\n",
    "    inputs = processor(images=image, return_tensors=\"pt\")\n",
    "    inputs.to(device)\n",
    "    with torch.no_grad():\n",
    "        img_ft = model.get_image_features(**inputs)\n",
    "    print(pt_img_features[n].to(\"cpu\")==img_ft.to(\"cpu\"))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True, True, True, True, True,\n",
      "         True, True, True, True, True, True, True, True]])\n"
     ]
    }
   ],
   "source": [
    "test_image(\"http://ecx.images-amazon.com/images/I/21iMxsyDBRL._SX300_.jpg\",0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "data": {
      "text/plain": "'http://ecx.images-amazon.com/images/I/21iMxsyDBRL._SX300_.jpg'"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_meta_df['imUrl'][0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "data": {
      "text/plain": "torch.Size([18357, 768])"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_features=torch.load(os.path.join(RAW_PATH,\"text_feat.pt\"))\n",
    "text_features.shape"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "data": {
      "text/plain": "18357"
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clean_sentences)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [
    {
     "data": {
      "text/plain": "BertModel(\n  (embeddings): BertEmbeddings(\n    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n    (position_embeddings): Embedding(512, 768)\n    (token_type_embeddings): Embedding(2, 768)\n    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n    (dropout): Dropout(p=0.1, inplace=False)\n  )\n  (encoder): BertEncoder(\n    (layer): ModuleList(\n      (0-11): 12 x BertLayer(\n        (attention): BertAttention(\n          (self): BertSelfAttention(\n            (query): Linear(in_features=768, out_features=768, bias=True)\n            (key): Linear(in_features=768, out_features=768, bias=True)\n            (value): Linear(in_features=768, out_features=768, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n          (output): BertSelfOutput(\n            (dense): Linear(in_features=768, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (intermediate): BertIntermediate(\n          (dense): Linear(in_features=768, out_features=3072, bias=True)\n          (intermediate_act_fn): GELUActivation()\n        )\n        (output): BertOutput(\n          (dense): Linear(in_features=3072, out_features=768, bias=True)\n          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (pooler): BertPooler(\n    (dense): Linear(in_features=768, out_features=768, bias=True)\n    (activation): Tanh()\n  )\n)"
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "random_i_feature=tokenizer(clean_sentences[-10:],max_length=512,truncation=True,padding=\"max_length\",return_tensors='pt')\n",
    "random_i_feature=model(**random_i_feature)\n",
    "random_i_feature=random_i_feature.last_hidden_state[:,0,:]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "outputs": [],
   "source": [
    "def test_text(a,b=None):\n",
    "    if b is not None:\n",
    "        temp=tokenizer(clean_sentences[a:b],max_length=512,truncation=True,padding=\"max_length\",return_tensors='pt')\n",
    "        temp=model(**temp)\n",
    "        temp=temp.last_hidden_state[:,0,:]\n",
    "        print(torch.allclose(temp,text_features[a:b],1e-1))\n",
    "        print(temp==text_features[a:b])\n",
    "\n",
    "        print(temp)\n",
    "        print(text_features[a:b])\n",
    "    else:\n",
    "        temp=tokenizer(clean_sentences[a],max_length=512,truncation=True,padding=\"max_length\",return_tensors='pt')\n",
    "        temp=model(**temp)\n",
    "        temp=temp.last_hidden_state[:,0,:]\n",
    "        print(torch.allclose(temp,text_features[a],1e-1))\n",
    "        print(temp==text_features[a])\n",
    "\n",
    "        print(temp)\n",
    "        print(text_features[a])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "tensor([[False, False, False,  ..., False, False, False],\n",
      "        [False, False, False,  ..., False, False, False],\n",
      "        [False, False, False,  ..., False, False, False],\n",
      "        [False, False, False,  ..., False, False, False]])\n",
      "tensor([[-0.5981, -0.5338,  0.0445,  ..., -0.1491,  0.0514, -0.1792],\n",
      "        [-0.4502, -0.0870,  0.1855,  ..., -0.4759,  0.4727,  0.2527],\n",
      "        [-0.8182, -0.4949, -0.4483,  ..., -0.6364,  0.0677, -0.0387],\n",
      "        [-0.6295, -0.4640, -0.1709,  ..., -0.6600, -0.0348, -0.0604]],\n",
      "       grad_fn=<SliceBackward0>)\n",
      "tensor([[-0.5981, -0.5338,  0.0445,  ..., -0.1491,  0.0514, -0.1792],\n",
      "        [-0.4502, -0.0870,  0.1855,  ..., -0.4759,  0.4727,  0.2527],\n",
      "        [-0.8182, -0.4949, -0.4484,  ..., -0.6364,  0.0677, -0.0387],\n",
      "        [-0.6294, -0.4640, -0.1709,  ..., -0.6600, -0.0348, -0.0604]],\n",
      "       grad_fn=<SliceBackward0>)\n"
     ]
    }
   ],
   "source": [
    "test_text(-5,-1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "outputs": [
    {
     "data": {
      "text/plain": "BertModel(\n  (embeddings): BertEmbeddings(\n    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n    (position_embeddings): Embedding(512, 768)\n    (token_type_embeddings): Embedding(2, 768)\n    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n    (dropout): Dropout(p=0.1, inplace=False)\n  )\n  (encoder): BertEncoder(\n    (layer): ModuleList(\n      (0-11): 12 x BertLayer(\n        (attention): BertAttention(\n          (self): BertSelfAttention(\n            (query): Linear(in_features=768, out_features=768, bias=True)\n            (key): Linear(in_features=768, out_features=768, bias=True)\n            (value): Linear(in_features=768, out_features=768, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n          (output): BertSelfOutput(\n            (dense): Linear(in_features=768, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (intermediate): BertIntermediate(\n          (dense): Linear(in_features=768, out_features=3072, bias=True)\n          (intermediate_act_fn): GELUActivation()\n        )\n        (output): BertOutput(\n          (dense): Linear(in_features=3072, out_features=768, bias=True)\n          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (pooler): BertPooler(\n    (dense): Linear(in_features=768, out_features=768, bias=True)\n    (activation): Tanh()\n  )\n)"
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "graph_llm",
   "language": "python",
   "display_name": "graph_llm"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}